diff --git a/.gitignore b/.gitignore index 1bf2c8f9..eec418ab 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ __pycache__/ /dist _version.py *.log + +# Local one-off helper scripts (not part of the project) +/tmp/ diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 9d6ad700..4c022a57 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -354,3 +354,47 @@ try: except ValueError as e: # Error handling... ``` +#### Use of `assert` in Runtime Code + +This package can legitimately be run under `python -O`, which strips every `assert` +statement from the bytecode. So the choice between `assert` and `raise` decides +whether a check exists in production at all. + +Use `assert` **only** for type-checker narrowing: a condition that the type system +cannot see but that is structurally guaranteed by the caller, where losing the check +costs nothing but a slightly less legible `AttributeError` on the following line. +Narrowing a schema-version union immediately after the caller has already selected +that version is the canonical example. + +Use an explicit `raise` for anything that is a *runtime invariant* — a condition that +could actually be false because of a race, a partially-completed construction, an +unexpected model shape, or a caller error. Concretely, prefer `raise` when any of +these is true: + +- The check is in a **public** method or property. A stripped assert turns a clear + library error into `AttributeError: 'NoneType' object has no attribute ...` raised + from inside this package. +- The code runs on a **background thread** — a pool worker, a `threading.Timer` + callback, or the stdout-forwarding path. An `AssertionError` there reaches only + `threading.excepthook`, so the failure is invisible and the surrounding work + (waiting on and reaping the subprocess, arming a grace timer) is silently skipped. +- The assert **is** the fix. Several checks in `_runner_base.py` exist specifically to + hold an invariant that a bug-fix introduced. Under `-O` such an assert reverts the + fix without changing any test result. +- Something **downstream would proceed on bad data** rather than stop, for example + materializing an embedded file from an object of an unrecognized schema. + +When a `raise` would be noisy in an already-defensive spot, prefer binding the value +once and handling `None` explicitly (return early, log, skip the notification) over +either an assert or a raise. That keeps the failure legible and the process owned. + +Both forms narrow types for mypy, so there is no typing cost to choosing `raise`: + +```py +# Narrowing only — fine as an assert. +assert isinstance(script, StepScript_2023_09) + +# A runtime invariant — must survive -O. +if not isinstance(file, EmbeddedFileText_2023_09): + raise RuntimeError(f"Unsupported embedded file model '{type(file).__name__}'.") +``` diff --git a/pyproject.toml b/pyproject.toml index e5277e8c..3430ccae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,13 @@ classifiers = [ "Topic :: Software Development :: Libraries" ] dependencies = [ - "openjd-model >= 0.10,< 0.12", + # 0.11.1 is the floor because it is the first release carrying the EXPR/ + # WRAP_ACTIONS model surface this package uses: FormatString.resolve_value and + # .whole_field_expression, evaluate_let_bindings, CancelationMethodDeferred, + # SymbolTable.expr_host_rules, and `let` on StepTemplate/StepScript. 0.11.0 has + # only StepTemplate.let, StepScript.let and SymbolTable.expr_types, so this + # package fails at import against it. + "openjd-model >= 0.11.1,< 0.12", "pywin32 >= 307; platform_system == 'Windows'", "psutil >= 5.9,< 7.3; platform_system == 'Windows'", ] @@ -149,6 +155,11 @@ addopts = [ "--cov-report=xml:build/coverage/coverage.xml", "--cov-report=term-missing", "--numprocesses=auto", + # loadgroup, not the default load: it honours @pytest.mark.xdist_group, which + # the tests that race a real subprocess against the wall clock rely on to run + # serially with respect to each other (see conftest.serial_process). Behaves + # exactly like `load` for every test that is not in a group. + "--dist=loadgroup", "--timeout=30", "--log-format=%(asctime)s.%(msecs)03d %(levelname)s %(filename)20s:%(lineno)-3s %(message)s", "--log-date-format=%H:%M:%S" diff --git a/src/openjd/sessions/_action_filter.py b/src/openjd/sessions/_action_filter.py index 9304c94e..25d43aac 100644 --- a/src/openjd/sessions/_action_filter.py +++ b/src/openjd/sessions/_action_filter.py @@ -15,6 +15,27 @@ __all__ = ("ActionMessageKind", "ActionMonitoringFilter", "redact_openjd_redacted_env_requests") +def _describe_exception(exc: BaseException) -> str: + """Render an exception for a log line without trusting it to be renderable. + + A consumer-supplied callback may raise an exception whose ``__str__`` or + ``__repr__`` itself raises. Every containment site in this module puts the + exception into a message, so rendering it naively re-creates the exact escape + the containment exists to prevent -- the exception then propagates from inside + the ``except`` block, out of ``filter()``, and into the thread forwarding the + subprocess's stdout. + """ + try: + return str(exc) + except Exception: + pass + try: + return repr(exc) + except Exception: + pass + return f"" + + def redact_openjd_redacted_env_requests(command_str: str) -> str: """Redact sensitive information in command strings before they're processed by the regular redaction mechanism. @@ -78,11 +99,27 @@ class ActionMessageKind(Enum): openjd_env_actions_filter_matcher = re.compile(openjd_env_actions_filter_regex) # A regex for matching the assignment of a value to an environment variable -envvar_set_regex_str = "^[A-Za-z_][A-Za-z0-9_]*" "=" ".*$" # Variable name -envvar_set_regex_json = '^(")?[A-Za-z_][A-Za-z0-9_]*' "=" ".*$" # Variable name +# +# Note on the anchors: these use \Z, not $, because `$` also matches immediately +# before a trailing newline -- so an `$`-anchored name pattern accepts "FOO\n", +# a name no operating system can hold. \Z matches only at the true end of the +# string. +# +# Honest scope: this is defence in depth, not a fix for a reachable defect. The +# outer `filter_regex` below captures the message body with `(.+)$`, and `.` never +# crosses a newline, so a trailing newline is already stripped before any name +# reaches these patterns. \Z is used so that these patterns remain correct on +# their own terms if that outer capture ever changes. +# +# The VALUE half deliberately stays permissive: a multi-line value delivered +# through the JSON form (openjd_env: "FOO=a\nb") is supported, tested behaviour, +# so the separator hygiene a value needs belongs at each serialization boundary, +# not here. +envvar_set_regex_str = "^[A-Za-z_][A-Za-z0-9_]*" "=" ".*\\Z" # Variable name +envvar_set_regex_json = '^(")?[A-Za-z_][A-Za-z0-9_]*' "=" ".*\\Z" # Variable name envvar_set_matcher_str = re.compile(envvar_set_regex_str) envvar_set_matcher_json = re.compile(envvar_set_regex_json) -envvar_unset_regex = "^[A-Za-z_][A-Za-z0-9_]*$" +envvar_unset_regex = "^[A-Za-z_][A-Za-z0-9_]*\\Z" envvar_unset_matcher = re.compile(envvar_unset_regex) @@ -191,7 +228,7 @@ def _redactions_enabled(self) -> bool: or "REDACTED_ENV_VARS" in self._revision_extensions.extensions ) - def apply_message_redaction(self, record: logging.LogRecord): + def apply_message_redaction(self, record: logging.LogRecord) -> None: """Redact the log message if it contains any substrings which have been registered for redaction Args: @@ -200,21 +237,36 @@ def apply_message_redaction(self, record: logging.LogRecord): # Check if we need to redact any sensitive values from the log message if (self._redacted_values or self._redacted_lines) and isinstance(record.msg, str): - # If we have args, first do string formatting, then redact - try: - record.msg = record.msg % record.args - record.args = () # Clear args since we've done the formatting - except Exception: - # If string formatting fails, fall back to just redacting the message - LOG.warning( - "Failed to format log message for redaction. Proceeding with redaction on unformatted message." - ) + # Fold `args` into `msg` and clear them UNCONDITIONALLY before + # scanning (R5-1 fix). The redaction below only ever inspects + # `record.msg`, and a downstream handler calls record.getMessage(), + # which re-runs `msg % args`. So any path that leaves `args` + # populated re-interpolates the un-scanned original into the emitted + # line -- a secret carried in `args` would be emitted verbatim. The + # `finally` is the load-bearing part: `args` must end up empty on the + # failure path too, not just on the success path. + if record.args: + try: + record.msg = record.msg % record.args + except Exception: + # The record's own formatting is broken, so getMessage() + # would raise inside the handler and Python's logging would + # dump the raw args to stderr, outside this filter's reach. + # Fold them in textually instead: the record stays emittable + # AND every byte of it goes through the scan below. + record.msg = f"{record.msg} {record.args!r}" + LOG.warning( + "Failed to format log message for redaction. Redacting the message with its " + "arguments appended instead.", + extra=LogExtraInfo(openjd_log_content=LogContent.COMMAND_OUTPUT), + ) + finally: + record.args = () # Check if the entire message matches a line in the redacted_lines set if record.msg in self._redacted_lines: record.msg = "*" * 8 - record.args = () - return True + return # Find all segments that need redaction segments_to_redact = [] @@ -254,7 +306,6 @@ def apply_message_redaction(self, record: logging.LogRecord): for start, end in reversed(merged_segments): msg_chars[start:end] = list("*" * 8) # Always use 8 asterisks for redaction record.msg = "".join(msg_chars) - record.args = () def filter(self, record: logging.LogRecord) -> bool: """Called automatically by Python's logging subsystem when a log record @@ -323,6 +374,27 @@ def filter(self, record: logging.LogRecord) -> bool: record.msg = record.msg + f" -- ERROR: {str(e)}" # There was an error. Don't suppress the message from the log. return True + except Exception as e: + # R5-2 fix: `handler` invokes the consumer-supplied callback, + # which may raise anything. Anything other than ValueError + # used to propagate out of filter(), into Python's logging + # machinery, and into the stdout pump thread that called + # logger.info() -- killing the pump, silently dropping the + # rest of the subprocess's output, and leaving the child + # unreaped. Same isolation as the Session's terminal + # callbacks; keep the record in the log so the failure is + # visible in the action's own output. + # + # _describe_exception, not f"{e}": rendering a hostile + # exception can itself raise, which would re-open this very + # escape from inside the handler for it. + detail = _describe_exception(e) + LOG.error( + f"Open Job Description: Exception in the action message callback: {detail}", + extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), + ) + record.msg = record.msg + f" -- ERROR: {detail}" + return True return not self._suppress_filtered # Check for "almost" matching openjd_env and openjd_unset_env commands @@ -335,13 +407,46 @@ def filter(self, record: logging.LogRecord) -> bool: record.msg = record.msg + f" -- ERROR: {err_message}" # Callback to cancel the action and mark it as FAILED - self._callback(ActionMessageKind.FAIL, err_message, True) + # R5-2 fix: isolated for the same reason as the handler call + # above -- this is the one consumer-callback invocation in this + # method that is not routed through `handler`. + self._invoke_callback(ActionMessageKind.FAIL, err_message, True) return True return True finally: - # Always check for redaction before returning - self.apply_message_redaction(record) + # Always check for redaction before returning. Fail closed: if the + # redaction control itself raises, blank the message rather than + # letting an unscanned record through (and rather than letting the + # exception reach the stdout pump thread). + try: + self.apply_message_redaction(record) + except Exception as e: + record.msg = "*" * 8 + record.args = () + LOG.error( + "Open Job Description: Log redaction failed; message suppressed: " + f"{_describe_exception(e)}", + extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), + ) + + def _invoke_callback(self, kind: ActionMessageKind, value: Any, cancel_and_fail: bool) -> None: + """Invoke the consumer-supplied callback, isolating any exception it + raises (R5-2). + + This filter runs on the thread that is forwarding the subprocess's stdout + to the log, so an exception escaping here unwinds + ``LoggingSubprocess.run()`` before the child is waited on. Consumer bugs + must not cost us the output stream or process ownership. + """ + try: + self._callback(kind, value, cancel_and_fail) + except Exception as e: + LOG.error( + "Open Job Description: Exception in the action message callback: " + f"{_describe_exception(e)}", + extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), + ) def _handle_progress(self, message: str) -> None: """Local handling of Progress messages. Processes the message and then @@ -404,33 +509,28 @@ def _parse_env_variable(self, message: str) -> tuple[str, str, bool, int, Option if equals_position == -1: return "", "", False, -1, None - # Check if the message is valid - is_valid = envvar_set_matcher_str.match(message) or envvar_set_matcher_json.match(message) - - if not is_valid: + # Check if the message is valid. Bound once: the plain-string match decides + # both validity and which parse branch to take below. + str_match = envvar_set_matcher_str.match(message) + if not (str_match or envvar_set_matcher_json.match(message)): return "", "", False, equals_position, None # Parse the variable name and value - try: - original_value = None - if envvar_set_matcher_str.match(message): - name, _, value = message.partition("=") - else: - # Handle JSON format - try: - # Store the original value before JSON parsing - original_value = message[equals_position + 1 :] - message_json_str = json.loads(message) - name, _, value = message_json_str.partition("=") - except json.JSONDecodeError as e: - raise ValueError( - f"Unterminated string starting at: line {e.lineno} column {e.colno} (char {e.pos})" - ) - return name, value, True, equals_position, original_value - except json.JSONDecodeError as e: - raise ValueError( - f"Unterminated string starting at: line {e.lineno} column {e.colno} (char {e.pos})" - ) + original_value = None + if str_match: + name, _, value = message.partition("=") + else: + # Handle JSON format + try: + # Store the original value before JSON parsing + original_value = message[equals_position + 1 :] + message_json_str = json.loads(message) + name, _, value = message_json_str.partition("=") + except json.JSONDecodeError as e: + raise ValueError( + f"Unterminated string starting at: line {e.lineno} column {e.colno} (char {e.pos})" + ) + return name, value, True, equals_position, original_value def _handle_env_error(self, error_message: str, is_redacted: bool = False) -> None: """Handle errors in environment variable processing. diff --git a/src/openjd/sessions/_embedded_files.py b/src/openjd/sessions/_embedded_files.py index b5444d9a..8f8d26b8 100644 --- a/src/openjd/sessions/_embedded_files.py +++ b/src/openjd/sessions/_embedded_files.py @@ -66,6 +66,48 @@ def _convert_line_endings(data: str, end_of_line: Optional[str]) -> str: return data +def chown_group(path: Path, group: str) -> None: + """Set ``path``'s group ownership, reporting failure as ``OSError``. + + ``shutil.chown`` raises ``LookupError`` when the group name does not resolve, + and ``LookupError`` is neither ``OSError`` nor ``ValueError``. Every handler + around a file-materialization or session-directory failure in this package + catches some combination of ``OSError``/``ValueError``/``RuntimeError``, so an + unknown group escaped all of them and surfaced as a bare ``LookupError`` out + of the public Session API. ``PosixSessionUser`` does not validate its + ``group``, so a caller only has to pass a group that does not exist. + + Translated here, at the two call sites, rather than by widening each handler: + a group that cannot be resolved *is* a failure to change ownership, the + callers already treat that as ``OSError``, and this way a handler added later + is covered without anyone having to remember. + """ + try: + chown(path, group=group) + except LookupError as err: + raise OSError(f"Could not set the group of {str(path)} to '{group}': {err}") from err + + +def _unsupported_embedded_file_model(file: EmbeddedFileType) -> RuntimeError: + """The error for an embedded file from a schema this class does not implement. + + Returned rather than raised so that call sites can keep the + ``if not isinstance(...): raise`` shape, which narrows the type for mypy + exactly as an ``assert isinstance`` did while still being enforced under + ``python -O`` (R5-6). These three checks are genuine runtime invariants, not + type-checker narrowing: with them stripped, the call sites would read + ``.filename``/``.data``/``.runnable`` off an object of an unknown shape and + materialize a file from whatever they happened to find, instead of stopping. + + When a new schema version is added, this is the seam that grows the + per-version dispatch. + """ + return RuntimeError( + f"Unsupported embedded file model '{type(file).__name__}'. This runtime implements the " + "2023-09 schema only." + ) + + def _validate_embedded_filename(filename: str) -> None: """Validate that an embedded file's ``filename`` is a single path component (a basename) with no directory pathing, as required by the OpenJD @@ -140,7 +182,7 @@ def write_file_for_user( if user is not None: user = cast(PosixSessionUser, user) # Set the group of the file - chown(filename, group=user.group) + chown_group(filename, user.group) # Update the permissions to include the group after the group is changed # Note: Only after changing group for security in case the group-ownership # change fails. @@ -209,11 +251,40 @@ def __init__( self._user = user def materialize(self, files: EmbeddedFilesListType, symtab: SymbolTable) -> None: - if self._scope == EmbeddedFilesScope.ENV: - self._logger.info("Writing embedded files for Environment to disk.") - else: - self._logger.info("Writing embedded files for Task to disk.") - + records = self.allocate_file_paths(files, symtab) + self.write_file_contents(records, symtab) + + def allocate_file_paths( + self, files: EmbeddedFilesListType, symtab: SymbolTable + ) -> list[_FileRecord]: + """Allocate the on-disk paths for the embedded files and define their + ``Env.File.*``/``Task.File.*`` symbols in ``symtab``, without writing + the file contents. + + Splitting allocation from :meth:`write_file_contents` lets the runner + evaluate EXPR ``let`` bindings between the two phases (RFC 0005): a + file's *path* never depends on ``let`` values (``filename`` is a plain + string), so the ``Env.File.*``/``Task.File.*`` symbols are available + to the bindings, while a file's ``data`` is written afterwards so it + can reference let-bound values. Mirrors the openjd-rs runners. + """ + self._log_scoped("Allocating embedded file paths for") + records = self.allocate_records(files) + self._define_symbols(records, symtab) + return records + + def allocate_records(self, files: EmbeddedFilesListType) -> list["_FileRecord"]: + """Allocate the on-disk paths for the embedded files, without logging + or defining any symbols. + + Used by the Session to pre-allocate a wrap environment's file records + once (RFC 0008); each wrap-hook invocation then defines the symbols + and logs through :meth:`register_file_paths` against its own symbol + table. + + Raises: + RuntimeError: If a file path could not be allocated. + """ try: records = list[_FileRecord]() # Generate the symbol table values and filenames @@ -221,18 +292,54 @@ def materialize(self, files: EmbeddedFilesListType, symtab: SymbolTable) -> None # Raises: OSError symbol, filename = self._get_symtab_entry(file) records.append(_FileRecord(symbol=symbol, filename=filename, file=file)) + return records + except (OSError, ValueError) as err: + raise RuntimeError(f"Could not write embedded file: {err}") - # Add symbols to the symbol table - for record in records: - symtab[record.symbol] = str(record.filename) - self._logger.info( - f"Mapping: {record.symbol} -> {record.filename}", - extra=LogExtraInfo( - openjd_log_content=LogContent.FILE_PATH | LogContent.PARAMETER_INFO - ), - ) + def register_file_paths(self, records: list["_FileRecord"], symtab: SymbolTable) -> None: + """Define the ``Env.File.*``/``Task.File.*`` symbols in ``symtab`` + for already-allocated records, without allocating new paths. + + Used to reuse a wrap environment's embedded-file records across + wrap-hook invocations (RFC 0008): the on-disk paths are allocated + once so the symbols stay stable and unnamed files do not accumulate, + while the contents are re-resolved and rewritten per invocation via + :meth:`write_file_contents`. + """ + self._log_scoped("Reusing embedded file paths for") + self._define_symbols(records, symtab) + + def _log_scoped(self, message: str) -> None: + """Log ``message`` qualified by this collection's scope. + + The phase messages are kept accurate on purpose: allocation reserves + paths, registration only defines symbols, and only + :meth:`write_file_contents` writes content. + """ + scope = "Environment" if self._scope == EmbeddedFilesScope.ENV else "Task" + self._logger.info(f"{message} {scope}.") + + def _define_symbols(self, records: list["_FileRecord"], symtab: SymbolTable) -> None: + # Add symbols to the symbol table. For EXPR evaluation the + # Env.File.*/Task.File.* symbols are host-format path values + # (property access like `.parent` works), matching openjd-rs; + # the legacy (non-EXPR) interpolation path ignores the type and + # keeps the string form. + for record in records: + symtab[record.symbol] = str(record.filename) + symtab.expr_types[record.symbol] = "PATH" + self._logger.info( + f"Mapping: {record.symbol} -> {record.filename}", + extra=LogExtraInfo( + openjd_log_content=LogContent.FILE_PATH | LogContent.PARAMETER_INFO + ), + ) - # Write the files to disk. + def write_file_contents(self, records: list["_FileRecord"], symtab: SymbolTable) -> None: + """Resolve each allocated file's ``data`` against ``symtab`` and write + it to disk. See :meth:`allocate_file_paths`.""" + self._log_scoped("Writing embedded files to disk for") + try: for record in records: # Raises: OSError self._materialize_file(record.filename, record.file, symtab) @@ -253,7 +360,8 @@ def _find_value_prefix(self, file: EmbeddedFileType) -> str: """ # When adding a new schema, start this method with a check for which # model 'file' belongs to -- that'll tell us the schema version. - assert isinstance(file, EmbeddedFileText_2023_09) + if not isinstance(file, EmbeddedFileText_2023_09): + raise _unsupported_embedded_file_model(file) if self._scope == EmbeddedFilesScope.ENV: return ValueReferenceConstants_2023_09.ENV_FILE_PREFIX.value @@ -274,7 +382,8 @@ def _get_symtab_entry(self, file: EmbeddedFileType) -> tuple[str, Path]: value - The absolute filename of the file to manifest. """ - assert isinstance(file, EmbeddedFileText_2023_09) + if not isinstance(file, EmbeddedFileText_2023_09): + raise _unsupported_embedded_file_model(file) # Figure out what filename to use for the given embedded file. # This will either be provided in the given 'file' or we will @@ -314,7 +423,8 @@ def _materialize_file( Make the file executable if the file settings indicate that we should. """ - assert isinstance(file, EmbeddedFileText_2023_09) + if not isinstance(file, EmbeddedFileText_2023_09): + raise _unsupported_embedded_file_model(file) execute_permissions = 0 if file.runnable: diff --git a/src/openjd/sessions/_linux/_sudo.py b/src/openjd/sessions/_linux/_sudo.py index 43a1fa8c..fccfef88 100644 --- a/src/openjd/sessions/_linux/_sudo.py +++ b/src/openjd/sessions/_linux/_sudo.py @@ -10,6 +10,12 @@ from .._logging import LoggerAdapter, LogContent, LogExtraInfo from .._os_checker import is_posix, is_linux +PGREP_NO_MATCH = 1 +"""``pgrep``'s exit status for "no processes matched". + +Distinct from a genuine failure (>=2 for a usage or operational error). See +:func:`find_child_process_id_pgrep`.""" + class FindSignalTargetError(Exception): """Exception when unable to detect the signal target""" @@ -44,7 +50,19 @@ def find_sudo_child_process_group_id( # send SIGKILL signals to the subprocess of sudo start = time.monotonic() now = start - sudo_pgid = os.getpgid(sudo_process.pid) + try: + sudo_pgid = os.getpgid(sudo_process.pid) + except ProcessLookupError: + # Sibling of the same guard in LoggingSubprocess.run(): `sudo` itself has + # already been reaped, so there is no group to find. Return the "unknown" + # value this function already uses for a child that exits mid-scan + # (below) rather than letting ESRCH escape to a caller that is only + # looking for a signal target. + logger.debug( + f"sudo process {sudo_process.pid} exited before its process group could be determined.", + extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), + ) + return None # Repeatedly scan for child processes # @@ -53,18 +71,36 @@ def find_sudo_child_process_group_id( # quickly and we may never find the child process. sudo_child_pid: Optional[int] = None sudo_child_pgid: Optional[int] = None + last_scan_error: Optional[FindSignalTargetError] = None try: while now - start < timeout_seconds: if not sudo_child_pid: - if is_linux(): - sudo_child_pid = find_sudo_child_process_id_procfs( - sudo_pid=sudo_process.pid, - logger=logger, - ) - else: - sudo_child_pid = find_child_process_id_pgrep( - sudo_pid=sudo_process.pid, - ) + try: + if is_linux(): + sudo_child_pid = find_sudo_child_process_id_procfs( + sudo_pid=sudo_process.pid, + logger=logger, + ) + else: + sudo_child_pid = find_child_process_id_pgrep( + sudo_pid=sudo_process.pid, + ) + except FindSignalTargetError as e: + # A single bad scan must not end the retries. Everything this + # loop is racing is transient by nature: the procfs scan sees + # more than one child while sudo's fork is still settling, and + # a `pgrep` invocation can fail for reasons that do not recur. + # This `except` used to sit outside the `while`, so the first + # such observation aborted the whole search and the process + # group was never recorded -- leaving a later cancel with + # nothing to signal. + # + # Remembered rather than discarded so that a timeout can say + # what kept going wrong. + # + # No `sudo_child_pid = None` here: the scan only runs when it + # is already falsy, so the assignment would be unfalsifiable. + last_scan_error = e if sudo_child_pid: try: @@ -84,7 +120,8 @@ def find_sudo_child_process_group_id( time.sleep(min(0.05, timeout_seconds - (now - start))) now = time.monotonic() if not sudo_child_pid or not sudo_child_pgid: - raise FindSignalTargetError("unable to detect subprocess before timeout") + detail = f" (last scan error: {last_scan_error})" if last_scan_error else "" + raise FindSignalTargetError(f"unable to detect subprocess before timeout{detail}") except FindSignalTargetError as e: logger.warning( f"Unable to determine signal target: {e}", @@ -143,8 +180,29 @@ def find_child_process_id_pgrep( stdin=DEVNULL, text=True, ) + if pgrep_result.returncode == PGREP_NO_MATCH: + # "No processes matched" is the expected answer for most of the retry + # window, not an error: sudo has forked but the kernel has not finished + # creating the workload yet. Returning None keeps the caller's retry loop + # going, which is what the procfs implementation of this same lookup + # already does for the identical situation. + # + # Raising here instead aborted the whole scan on the first empty poll, + # because the caller's `except FindSignalTargetError` sits OUTSIDE its + # retry `while`. Measured on macOS: the very first scan at ~25ms returned + # None for a child that appeared at ~300ms and was found correctly once it + # existed. So on every non-Linux POSIX host, a cross-user launch recorded + # no process group at all -- and a later cancel then had nothing to + # signal. + return None if pgrep_result.returncode != 0: - raise FindSignalTargetError("Unable to query child processes of sudo process") + # stderr is merged into stdout by the caller's `stderr=STDOUT`, so this is + # the only place the reason is available. Without it the message named an + # exit code and nothing else, which is not enough to act on. + raise FindSignalTargetError( + f"Unable to query child processes of sudo process (pgrep exited " + f"{pgrep_result.returncode}): {pgrep_result.stdout.strip()!r}" + ) results = pgrep_result.stdout.splitlines() if len(results) > 1: raise FindSignalTargetError(f"Expected a single child process of sudo, but found {results}") diff --git a/src/openjd/sessions/_path_mapping.py b/src/openjd/sessions/_path_mapping.py index b84178ff..48e1cad0 100644 --- a/src/openjd/sessions/_path_mapping.py +++ b/src/openjd/sessions/_path_mapping.py @@ -1,30 +1,83 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +import re from dataclasses import dataclass, fields from enum import Enum from os import name as os_name from pathlib import PurePath, PurePosixPath, PureWindowsPath +from string import ascii_lowercase, ascii_uppercase +from typing import Optional, Union + +_ASCII_LOWER_TABLE = str.maketrans(ascii_uppercase, ascii_lowercase) +"""Translation table for an ASCII-only case fold. + +RFC 3986 makes a URI's scheme and authority case-insensitive over ASCII only, +which is what openjd-rs implements (``str::eq_ignore_ascii_case``). ``str.lower()`` +would additionally fold non-ASCII characters — U+212A KELVIN SIGN lowers to ASCII +``k``, so ``s3://bucket\u212a`` would match ``s3://bucketk`` here but not in +openjd-rs. A translation table is also safer than ``str.lower()``/``str.casefold()`` +because it cannot change the string's length (``"\u0130".lower()`` yields two +characters), and :meth:`PathMappingRule._apply_uri` relies on offsets into the +string.""" + + +def _ascii_lower(value: str) -> str: + """Lower-case the ASCII letters in ``value``, leaving everything else alone.""" + return value.translate(_ASCII_LOWER_TABLE) + + +_URI_SOURCE_RE = re.compile(r"[A-Za-z][A-Za-z0-9+.\-]*://") +"""A URI-format rule's ``source_path`` must start with ``://``. + +Mirrors the scheme grammar of RFC 3986 §3.1, which is what the EXPR engine's +URI parser accepts. Validated in the constructor so a malformed rule is rejected +where the value can be named.""" class PathFormat(str, Enum): POSIX = "POSIX" WINDOWS = "WINDOWS" + # RFC 0006 §2.3.2 (EXPR extension): URI-form source paths + # (e.g. "s3://bucket/prefix") that map to local filesystem paths. + URI = "URI" @dataclass(frozen=True) class PathMappingRule: source_path_format: PathFormat - source_path: PurePath + # URI-format rules keep the source as the raw string: URIs are not + # filesystem paths, and PurePath normalization would corrupt the + # "scheme://" separator. + source_path: Union[PurePath, str] destination_path: PurePath def __init__( - self, *, source_path_format: PathFormat, source_path: PurePath, destination_path: PurePath + self, + *, + source_path_format: PathFormat, + source_path: Union[PurePath, str], + destination_path: PurePath, ): if source_path_format == PathFormat.POSIX: if not isinstance(source_path, PurePosixPath): raise ValueError( "Path mapping rule source_path_format does not match source_path type" ) + elif source_path_format == PathFormat.URI: + if not isinstance(source_path, str): + raise ValueError( + "Path mapping rule source_path must be a string for the URI source_path_format" + ) + # Validate the shape here, where the offending value can be named. + # The EXPR engine requires a real "scheme://" for a URI rule, and + # these rules also reach it via the session's host context — without + # this check a single-slash typo in a pathmapping-1.0 document + # surfaces as an opaque ValueError out of Session() instead. + if _URI_SOURCE_RE.match(source_path) is None: + raise ValueError( + "Path mapping rule source_path for the URI source_path_format must " + f"begin with '://', got {source_path!r}" + ) else: if not isinstance(source_path, PureWindowsPath): raise ValueError( @@ -49,9 +102,12 @@ def from_dict(rule: dict[str, str]) -> "PathMappingRule": raise ValueError(f"Path mapping rule requires the following fields: {field_names}") source_path_format = PathFormat(rule["source_path_format"].upper()) - source_path: PurePath + source_path: Union[PurePath, str] if source_path_format == PathFormat.POSIX: source_path = PurePosixPath(rule["source_path"]) + elif source_path_format == PathFormat.URI: + # Keep URIs verbatim; PurePath would collapse "scheme://". + source_path = rule["source_path"] else: source_path = PureWindowsPath(rule["source_path"]) destination_path = PurePath(rule["destination_path"]) @@ -76,6 +132,66 @@ def to_dict(self) -> dict[str, str]: "destination_path": str(self.destination_path), } + @staticmethod + def _uri_path_start(uri: str) -> Optional[int]: + """Byte offset where the path component begins in a URI (after + ``scheme://authority``), or None when there is no ``://`` separator. + Mirrors openjd-rs's ``uri_path_start``.""" + scheme_sep = uri.find("://") + if scheme_sep == -1: + return None + authority_start = scheme_sep + 3 + slash = uri.find("/", authority_start) + return len(uri) if slash == -1 else slash + + def _apply_uri(self, path: str) -> tuple[bool, str]: + """Apply a URI-format rule, mirroring openjd-rs's ``apply_uri``. + + Per RFC 3986 the scheme and authority match case-insensitively while + the path portion matches case-sensitively, on whole path components. + The result is a local path in the host's format. + """ + sep = "/" if os_name == "posix" else "\\" + source = str(self.source_path) + src_path_start = self._uri_path_start(source) + src_path_start = 0 if src_path_start is None else src_path_start + inp_path_start = self._uri_path_start(path) + inp_path_start = 0 if inp_path_start is None else inp_path_start + + # Scheme+authority must match case-insensitively, over ASCII only — + # see _ASCII_LOWER_TABLE for why this is not str.lower(). + if _ascii_lower(path[:inp_path_start]) != _ascii_lower(source[:src_path_start]): + return False, path + # Path portion must match case-sensitively, on a component boundary. + src_path = source[src_path_start:] + inp_path = path[inp_path_start:] + if not inp_path.startswith(src_path): + return False, path + remainder = inp_path[len(src_path) :] + if remainder and not remainder.startswith("/"): + return False, path + + child_parts = remainder[1:].split("/") if remainder else [] + result = str(self.destination_path) + for part in child_parts: + result += sep + part + if path.endswith("/") and not result.endswith(sep): + result += sep + return True, result + + def _source_path_component_count(self) -> int: + """Number of components in the source path, used to order rules from + most to least specific. For URI sources the ``scheme://authority`` + counts as one component plus one per path segment.""" + if isinstance(self.source_path, PurePath): + return len(self.source_path.parts) + source = str(self.source_path) + path_start = self._uri_path_start(source) + if path_start is None: + return 1 + path_portion = source[path_start:].strip("/") + return 1 + (len(path_portion.split("/")) if path_portion else 0) + def apply(self, *, path: str) -> tuple[bool, str]: """Applies the path mapping rule on the given path, if it matches the rule. Does not collapse ".." since symbolic paths could be used. @@ -83,18 +199,35 @@ def apply(self, *, path: str) -> tuple[bool, str]: Returns: tuple[bool, str] - indicating if the path matched the rule and the resulting mapped path. If it doesn't match, then it returns the original path unmodified. """ + if self.source_path_format == PathFormat.URI: + return self._apply_uri(path) + source_path = self.source_path + # Unreachable: the constructor guarantees non-URI rules carry PurePath + # sources. An assert rather than a raise because its only job here is to + # narrow the Union for mypy. + assert isinstance(source_path, PurePath) pure_path: PurePath if self.source_path_format == PathFormat.POSIX: pure_path = PurePosixPath(path) + if not pure_path.is_relative_to(source_path): + return False, path else: pure_path = PureWindowsPath(path) - - if not pure_path.is_relative_to(self.source_path): - return False, path - - remapped_parts = ( - self.destination_path.parts + pure_path.parts[len(self.source_path.parts) :] - ) + # Windows paths match case-insensitively, but over ASCII only — + # PureWindowsPath.is_relative_to() folds with str.lower(), which also + # folds non-ASCII characters (U+212A KELVIN SIGN lowers to ASCII + # 'k'), so a homoglyph in a submitted path would remap here while the + # EXPR engine's apply_path_mapping() and openjd-rs, which both use an + # ASCII-only fold, leave it alone. RFC 0006 2.3.2 says the two are + # the same transformation, so compare components ourselves. + source_parts = source_path.parts + path_parts = pure_path.parts + if len(path_parts) < len(source_parts) or any( + _ascii_lower(sp) != _ascii_lower(pp) for sp, pp in zip(source_parts, path_parts) + ): + return False, path + + remapped_parts = self.destination_path.parts + pure_path.parts[len(source_path.parts) :] if os_name == "posix": result = str(PurePosixPath(*remapped_parts)) if self._has_trailing_slash(self.source_path_format, path): @@ -109,5 +242,6 @@ def apply(self, *, path: str) -> tuple[bool, str]: def _has_trailing_slash(self, os: PathFormat, path: str) -> bool: if os == PathFormat.POSIX: return path.endswith("/") - else: - return path.endswith("\\") + # A Windows path may use either separator, and the EXPR engine (and + # openjd-rs) accept both here. + return path.endswith("\\") or path.endswith("/") diff --git a/src/openjd/sessions/_runner_base.py b/src/openjd/sessions/_runner_base.py index 27083937..1d8e917a 100644 --- a/src/openjd/sessions/_runner_base.py +++ b/src/openjd/sessions/_runner_base.py @@ -2,6 +2,7 @@ import json import os +import re import stat import shlex from abc import ABC, abstractmethod @@ -11,14 +12,24 @@ from enum import Enum from pathlib import Path from threading import Lock, Timer -from typing import Callable, Optional, Sequence, Type, cast +from typing import Any, Callable, Literal, Optional, Sequence, Type, cast from types import TracebackType from tempfile import mkstemp from openjd.model import SymbolTable from openjd.model import FormatStringError +from openjd.model import evaluate_let_bindings + +# The EXPR engine's typed value. Imported concretely (rather than duck-typed +# with getattr) so that a model API change fails loudly at import time instead +# of silently mis-classifying every optional integer field as "omitted". +# openjd.expr ships in the same distribution as openjd.model, which this module +# already hard-imports unreleased API from. +from openjd.expr import ExprValue, TypeCode from openjd.model.v2023_09 import Action as Action_2023_09 -from ._embedded_files import EmbeddedFiles, EmbeddedFilesScope, write_file_for_user +from openjd.model.v2023_09 import CancelationMethodDeferred as CancelationMethodDeferred_2023_09 +from openjd.model.v2023_09 import CancelationMode as CancelationMode_2023_09 +from ._embedded_files import EmbeddedFiles, EmbeddedFilesScope, _FileRecord, write_file_for_user from ._logging import log_subsection_banner, LoggerAdapter, LogContent, LogExtraInfo from ._os_checker import is_posix from ._session_user import SessionUser @@ -32,9 +43,113 @@ "TerminateCancelMethod", "NotifyCancelMethod", "ScriptRunnerBase", + "apply_let_bindings", + "resolve_action_arg_values", + "resolve_effective_cancelation", + "resolve_optional_int_field", + "MAX_INT_FIELD_VALUE", + "MAX_LET_BINDING_LENGTH", + "MAX_SCHEDULABLE_TIMEOUT_SECONDS", + "POSIX_SHELL_NAME_RE", ) +POSIX_SHELL_NAME_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +"""The only shape a POSIX shell variable name may take (POSIX.1-2017 §3.235). + +Used to gate what ``_generate_command_shell_script`` is willing to emit an +``export``/``unset`` line for. Deliberately identical to the name grammar +``_action_filter`` enforces on ``openjd_env`` messages, so a variable defined by +a job's stdout and one supplied by the embedder are held to the same standard. +""" + +_STRICT_INT_RE = re.compile(r"[+-]?[0-9]+") +"""The exact integer grammar accepted for dynamically resolved integer fields +(FEATURE_BUNDLE_1 ``timeout`` / ``notifyPeriodInSeconds`` format strings): +an optional sign followed by ASCII digits, exactly what Rust's ``str::parse`` +accepts. See resolve_optional_int_field.""" + +MAX_INT_FIELD_VALUE = 2**63 - 1 +"""Largest value accepted for a dynamically resolved integer field. + +Three limits coincide here. openjd-rs's model rejects a literal ``timeout`` +above ``i64::MAX`` at parse time; its runtime parses a resolved one with +``str::parse::()`` (``runner/mod.rs`` resolve_action_timeout), which *fails* +rather than saturating; and the EXPR engine's integers are ``i64``, so a larger +value cannot round-trip through ``WrappedAction.Timeout`` (RFC 0008) at all. +Python's model has no upper bound, so the bound is applied here — to literal and +resolved values alike, which is what keeps a forwarded value behaving exactly as +it would have unwrapped.""" + +MAX_SCHEDULABLE_TIMEOUT_SECONDS = 2**62 // 10**9 +"""Largest action timeout we can actually enforce, in seconds (~146 years). + +Two distinct limits sit above this. :class:`datetime.timedelta` tops out at +999,999,999 days and raises ``OverflowError`` on construction. Separately, +:class:`threading.Timer` computes an absolute deadline in CPython's internal +64-bit nanosecond time representation, which overflows just above 2**63 +nanoseconds (measured: 9,223,372,036 s schedules, 9,223,372,037 s does not) — +that one raises on the timer's own thread, so the timeout silently never fires +rather than failing loudly. The bound here is deliberately well inside that +boundary rather than pinned to it, because the overflow is reported against the +platform's ``time_t`` and the exact threshold is not portable. + +A timeout beyond this is indistinguishable from "no timeout", which is what +openjd-rs's ``Duration``-based timer effectively provides at that magnitude, so +the action runs unbounded instead of failing. See _timeout_from_seconds.""" + +MAX_LET_BINDING_LENGTH = 4096 +"""Largest ``let`` binding string this runtime will parse, in characters. + +Not a spec limit. openjd-model's expression parser recurses per nesting level +with no depth guard of its own, so a deeply nested RHS overflows the C stack +and kills the interpreter with SIGBUS — no exception, no log line, no cleanup. +A ``let`` RHS is the only template-controlled expression source that is parsed +*after* model validation accepts it (format strings are parsed during +model_validate), so it is the only one a submission-time validator cannot +screen; the bound therefore has to be applied here, at the last gate before +the parse. + +Measured on this branch: a 26,001-character RHS parses safely, a 30,001- +character one crashes. 4096 is deliberately an order of magnitude inside that +boundary rather than pinned to it, because the true limit is the platform's +thread stack size and is not portable. openjd-rs bounds the same risk by +parsing on a worker thread with an explicit PARSER_THREAD_STACK_SIZE +(openjd-expr parse.rs) and its conformance suite requires that neither +implementation *crash* on a too-deep expression (test_expression_depth.rs) — +rejecting cleanly is conformant. + +Remove this in favour of the engine's own guard once the openjd-model pin +floor carries the parser-stack fix; keep the regression test either way.""" + + +def _over_range_message(description: str, value: int) -> str: + """Error text for a value that is a valid integer but too large to use. + + Distinct from the bounds message, because "must be a positive integer" reads + as nonsense for a value that plainly is one -- the problem is magnitude. + """ + return f"{description} must be at most {MAX_INT_FIELD_VALUE}, got '{value}'" + + +def _timeout_from_seconds(seconds: int, logger: LoggerAdapter) -> Optional[timedelta]: + """Convert a resolved timeout in seconds into an enforceable time limit. + + Returns ``None`` — no time limit — for a value too large to schedule (see + :data:`MAX_SCHEDULABLE_TIMEOUT_SECONDS`), so that an absurd-but-valid + timeout runs the action to completion the way openjd-rs does, rather than + raising ``OverflowError`` out of the public Session API. + """ + if seconds > MAX_SCHEDULABLE_TIMEOUT_SECONDS: + logger.warning( + f"Action timeout of {seconds} seconds is larger than this runtime can enforce; " + "the action will run without a time limit.", + extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), + ) + return None + return timedelta(seconds=seconds) + + class ScriptRunnerState(str, Enum): """State of a ScriptRunner.""" @@ -90,6 +205,257 @@ class NotifyCancelMethod(CancelMethod): """Amount of time after a SIGTERM to wait to do the SIGKILL""" +def resolve_action_arg_values(args: Optional[Sequence], symtab: SymbolTable) -> list[str]: + """Resolve an action's ``args`` field into its flat list of argument + strings (excluding the command). + + RFC 0005 §1.3.2 argument semantics, mirroring openjd-rs's + resolve_action_args: a whole-field expression argument resolves typed — + a null result skips the argument, a list result flattens inline (one + argument per element, rendered with the engine's display coercion), and + a scalar becomes a single argument. Multi-segment format strings and + legacy (non-EXPR) expressions resolve to their string form. + + Shared by the enforcement path (:meth:`ScriptRunnerBase._run_action`) + and the RFC 0008 ``WrappedAction.Args`` injection, so a wrap hook sees + exactly the arguments the wrapped action would have run with unwrapped. + + Raises: + FormatStringError: If an argument's expression cannot be resolved. + """ + resolved: list[str] = [] + if args is not None: + for arg in args: + try: + value = arg.resolve_value(symtab=symtab) + except FormatStringError: + # Mirror openjd-rs's resolve_action_args: when typed + # resolution fails (e.g. a legacy-parsed expression meeting + # a typed symbol value), fall back to plain string + # resolution — which raises FormatStringError itself if the + # argument is genuinely unresolvable. + resolved.append(arg.resolve(symtab=symtab)) + continue + if isinstance(value, str): + resolved.append(value) + elif value.is_null: + continue + elif value.type.type_code == TypeCode.LIST: + resolved.extend(str(element) for element in value) + else: + resolved.append(str(value)) + return resolved + + +def resolve_optional_int_field( + value: Any, + symtab: SymbolTable, + *, + ge: Optional[int] = None, + le: Optional[int] = None, + description: str, +) -> Optional[int]: + """Resolve an optional int-or-format-string field (e.g. an action's + ``timeout`` or a cancelation's ``notifyPeriodInSeconds``) into an + optional integer. + + - ``None`` (field omitted) stays ``None``. + - A literal ``int`` passes through unchecked: literal values were + bounds-checked by the static validator at parse time. + - A FormatString (FEATURE_BUNDLE_1) is resolved against ``symtab`` + using typed resolution. A whole-field expression that resolves to a + typed null is treated as if the field were not provided (``None`` — + the caller applies any positional schema default). Any other result + — including a genuine empty string — must be an integer within the + given bounds; the bounds apply here because format-string values + could not be checked at parse time. This matches the openjd-rs + runtime (resolve_action_timeout / resolve_notify_period_seconds), + which only treats an ExprValue::Null result as "field omitted" and + errors on an empty string. + + Raises: + ValueError: If the resolved value is not an integer, or violates + the ``ge``/``le`` bounds. + FormatStringError: If expression resolution itself fails. + """ + if value is None: + return None + if ge is not None and le is not None: + constraint = f"between {ge} and {le}" + elif ge == 1 and le is None: + constraint = "a positive integer" + elif ge is not None: + constraint = f"an integer >= {ge}" + else: + constraint = "an integer" + if isinstance(value, int): + # Literal values were bounds-checked by the static validator at parse + # time -- with one exception: the validator has no upper bound. Reject an + # over-range literal here, mirroring openjd-rs, whose model rejects it at + # parse time and whose runtime's str::parse fails rather than saturating. + if value > MAX_INT_FIELD_VALUE: + raise ValueError(_over_range_message(description, value)) + return value + # Typed resolution: a whole-field EXPR expression yields the engine's + # typed value, so a null result is distinguishable from a genuine empty + # string. Multi-segment and legacy (non-EXPR) format strings fall back + # to plain string resolution — correct, since typed nulls only exist + # under EXPR whole-field semantics (Template Schemas 5.3). + resolved_value = value.resolve_value(symtab=symtab) + if isinstance(resolved_value, ExprValue) and resolved_value.is_null: + return None + resolved = str(resolved_value) + # Strict ASCII integer grammar, matching the Rust runtime's str::parse + # (openjd-rs resolve_action_timeout / resolve_notify_period_seconds). + # Python's int() is more lenient — it accepts surrounding whitespace, + # digit-group underscores ("1_0" == 10), and non-ASCII decimal digits — + # all of which Rust rejects, so accepting them here would be a + # spec-observable divergence. + if _STRICT_INT_RE.fullmatch(resolved) is None: + raise ValueError(f"{description} must be {constraint}, got '{resolved}'") + result = int(resolved) + if result > MAX_INT_FIELD_VALUE: + # See MAX_INT_FIELD_VALUE: openjd-rs's str::parse fails rather than + # saturating, so an over-range value must fail here too rather than + # reach the timer. + raise ValueError(_over_range_message(description, result)) + if (ge is not None and result < ge) or (le is not None and result > le): + raise ValueError(f"{description} must be {constraint}, got '{result}'") + return result + + +def resolve_effective_cancelation( + cancelation: Any, symtab: SymbolTable +) -> tuple[Optional[str], Optional[int]]: + """Resolve an action's cancelation config against the symbol table into + an effective ``(mode, notify_period_seconds)`` pair. + + What is the problem this solves? + + Format strings are normally delay-processed: the parser stores "this is + a format string" and the value resolves right before the action runs. + But a cancelation's ``mode`` is the *schema selector* — the parser needs + it to know which object shape it is reading — while a forwarded value + like ``mode: "{{WrappedAction.Cancelation.Mode}}"`` (RFC 0008 + round-trip forwarding) only exists at run time. The model therefore + carries such a mode as :class:`CancelationMethodDeferred`, and *this* + function is where the deferred decision finally lands: with the + ``WrappedAction.*`` values in the symbol table, the mode expression + resolves to ``"TERMINATE"``, ``"NOTIFY_THEN_TERMINATE"``, or null — and + a null mode means the whole cancelation object is treated as never + declared (the runtime default applies). + + Returns: + (mode, notify_period_seconds) where ``mode`` is ``"TERMINATE"``, + ``"NOTIFY_THEN_TERMINATE"``, or ``None`` (no ```` + declared, or a deferred mode that resolved to null); and + ``notify_period_seconds`` is ``None`` when the field was omitted or + its whole-field expression resolved to null — the caller applies + the positional schema default. + + Raises: + ValueError: If a deferred mode resolves to anything other than the + two method names or null, if a resolved TERMINATE carries a + non-null notify period, or if a notify period does not resolve + to a positive integer. + FormatStringError: If expression resolution itself fails. + """ + + def resolve_period(period: Any) -> Optional[int]: + # Bounds mirror the static validator's on literal values (Template + # Schemas 5.3.2: 1..600); see resolve_optional_int_field. + return resolve_optional_int_field( + period, symtab, ge=1, le=600, description="notifyPeriodInSeconds" + ) + + if cancelation is None: + return (None, None) + if isinstance(cancelation, CancelationMethodDeferred_2023_09): + # Typed resolution. Null semantics apply only to a whole-field + # expression ("{{ ... }}" with no surrounding text, target type + # string? — Template Schemas 5.3), and resolve_value only yields a + # typed null for a whole-field EXPR expression; every other format + # string resolves to its plain string form. A format string that + # happens to resolve to the empty string is NOT null; it falls + # through to the "must resolve to..." error below (matching the + # openjd-rs runtime, which errors on any non-null, non-mode-name + # result). + mode_value = cancelation.mode.resolve_value(symtab=symtab) + if isinstance(mode_value, ExprValue) and mode_value.is_null: + # Null mode drops the ENTIRE cancelation object: mode is the + # object's required discriminator, so an "omitted" mode cannot + # leave a partial object behind. The action behaves exactly as + # if no were declared. + return (None, None) + mode = str(mode_value) + if mode == CancelationMode_2023_09.TERMINATE.value: + # Post-resolution the object must validate against the resolved + # variant's shape: TERMINATE admits no notify period. + if resolve_period(cancelation.notifyPeriodInSeconds) is not None: + raise ValueError( + "cancelation mode resolved to TERMINATE, which does not " + "accept notifyPeriodInSeconds" + ) + return (CancelationMode_2023_09.TERMINATE.value, None) + if mode == CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value: + return ( + CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value, + resolve_period(cancelation.notifyPeriodInSeconds), + ) + raise ValueError( + "cancelation mode must resolve to TERMINATE, NOTIFY_THEN_TERMINATE, " + f"or null; got '{mode}'" + ) + if cancelation.mode == CancelationMode_2023_09.TERMINATE: + return (CancelationMode_2023_09.TERMINATE.value, None) + # Direct attribute access, not getattr with a default: the mode above has + # already established this is a NOTIFY_THEN_TERMINATE object, which always + # carries the field. A getattr default would silently substitute the + # positional 30/120 s default for the author's period if the model ever + # renamed it. + return ( + CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value, + resolve_period(cancelation.notifyPeriodInSeconds), + ) + + +def apply_let_bindings(*, symtab: SymbolTable, let_bindings: list[str]) -> None: + """Evaluate EXPR ``let`` bindings (RFC 0005) and add them to ``symtab``. + + ``let_bindings`` is a script's ``let`` field: an ordered list of + ``"name = expression"`` strings. Each RHS is an EXPR expression evaluated + against the symbol table built so far (so later bindings can reference + earlier ones), and the engine's typed result is stored under the bound + name — a let-bound path stays a path for property access, and float + rendering fidelity is preserved — matching the Rust runtime's natively + typed symbol table. + + The runners evaluate bindings after embedded-file *path* allocation and + before file *contents* are written, so a binding may reference + ``Env.File.*``/``Task.File.*`` and a file's ``data`` may reference + let-bound values (mirroring openjd-rs's runner ordering). + + Raises: + ValueError (FormatStringError/ExpressionError): if a binding's + expression cannot be evaluated, or if a binding is too long to + parse safely (see MAX_LET_BINDING_LENGTH). + """ + # R4-1 fix: Guard against SIGBUS crash from parser stack overflow. Check + # length BEFORE calling evaluate_let_bindings, because the crash happens + # in native code with no exception — the process just dies. + for binding in let_bindings: + if len(binding) > MAX_LET_BINDING_LENGTH: + # Truncate the binding text in the error message, since it may be + # tens of KB — that's the whole point of rejecting it. + raise ValueError( + f"let binding {binding[:40]!r}... is {len(binding)} characters, " + f"which exceeds the maximum of {MAX_LET_BINDING_LENGTH}" + ) + # Single-sourced in openjd.model (parse-memoized; skips malformed + # bindings; raises ValueError naming the failing binding). + evaluate_let_bindings(symtab=symtab, let_bindings=let_bindings) + + class ScriptRunnerBase(ABC): """Base class for a runnable Environment or Step Script. Responsible for running a *single* Action, and optionally canceling it. @@ -153,6 +519,26 @@ class ScriptRunnerBase(ABC): """True iff the subprocess was canceled but action needs to be notified as FAILED. """ + _launched: bool + """True once a launch has been *attempted*, successfully or not. + + Backs the single-use guard in :meth:`_run`. Deliberately a latch of its own + rather than something inferred from :attr:`state`: a runner that tried to + launch and failed must never be reusable, and tying that rule to the state + classification made it break whenever the classification changed. + """ + + _pending_cancel: Optional[tuple[Optional[timedelta], bool]] + """A cancel that arrived before the subprocess existed, as + ``(time_limit, mark_action_failed)``. + + ``cancel()`` is a cross-thread API, so it can land during action setup — + between resolving the action and the subprocess actually being created. The + request is remembered here and applied by :meth:`_run` as soon as the + subprocess exists, so a cancel is never silently dropped (openjd-rs holds + the equivalent state in a sticky ``CancellationToken``). + """ + _runtime_limit: Optional[Timer] """The Timer that will fire when the currently running Action has exhausted its runtime limit. @@ -177,6 +563,20 @@ class ScriptRunnerBase(ABC): e.g. We failed to write embedded files before even trying to run the action. """ + _resolved_cancel_method: Optional[CancelMethod] + """The running action's effective cancel method, resolved by + :meth:`_run_action` right before subprocess launch — against the same + final (let/embedded-file enriched) symbol table the command and args + resolved with — and consumed by the runners' :meth:`cancel`. + + Resolving at launch matches the openjd-rs runtime (``run_action`` + resolves ``cancel_method_for_action`` up front): a cancelation whose + deferred mode or notify period cannot be resolved fails the action at + start rather than surfacing only if a cancel later occurs, and the + resolution scope is the action's own (a mode referencing a script-level + ``let`` binding resolves correctly). ``None`` until an action launches. + """ + def __init__( self, *, @@ -225,6 +625,9 @@ def __init__( # Will run at most the run futures self._pool = ThreadPoolExecutor(max_workers=1) self._state_override = None + self._resolved_cancel_method = None + self._pending_cancel = None + self._launched = False self._print_section_banner = True # Context manager for use in our tests @@ -243,7 +646,50 @@ def shutdown(self) -> None: """Performs a clean shutdown on the runner. This shutsdown the internal ThreadPoolExectutor. """ - self._pool.shutdown() + # F7 fix: Detect self-join. If shutdown() is called from the pool's + # worker thread (e.g., from _on_process_exit -> _action_callback -> + # cleanup -> runner.shutdown), calling _pool.shutdown(wait=True) would + # deadlock because the thread is trying to join itself. In that case, + # defer to a background thread or skip the wait. + import threading + + # Check if current thread is the pool's worker thread. The pool has + # max_workers=1, so if a future is running, _threads has one element. + pool_threads: set[threading.Thread] = getattr(self._pool, "_threads", set()) + current = threading.current_thread() + if current in pool_threads: + # We're inside the worker thread. Use wait=False to avoid deadlock. + # The pool will be garbage-collected eventually. + self._pool.shutdown(wait=False) + else: + self._pool.shutdown() + + def _fail_action(self, message: str) -> None: + """Fail the action through the normal failure path: surface the + failure reason to the customer via the action filter + (``openjd_fail``), set the FAILED state override, and invoke the + callback. The subprocess's future may not have been started yet, + but the Session still needs to know that the action is over. + """ + self._logger.info( + f"openjd_fail: {message}", + extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), + ) + self._state_override = ScriptRunnerState.FAILED + if self._callback is not None: + # R4-6 fix: Isolate consumer-callback exceptions. Same pattern as + # the completion callback in _on_process_exit. A consumer that + # raises must not turn a handled template failure into an exception + # escaping the public Session API. + try: + self._callback(ActionState.FAILED) + except Exception as exc: + self._logger.error( + f"Exception in action callback: {exc}", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO + ), + ) @abstractmethod def cancel( @@ -269,8 +715,19 @@ def state(self) -> ScriptRunnerState: return ScriptRunnerState.READY # Check on the state of the future for done/canceled first # If the future is done, then we have a terminal state. - assert self._run_future is not None # For the type checker - if self._run_future.done(): + # + # R5-6: this is a reachable invariant, not type-checker narrowing, so it + # is a plain read rather than an `assert`. `_run` assigns `_process` + # before submitting the future, so a submit that fails leaves the pair + # inconsistent -- and this is a *property*, on the path every consumer + # polls, where an AssertionError (or, under `python -O`, an + # AttributeError) makes the runner permanently unreadable. A launched + # process with no future has not started running, so report READY and let + # the caller's own error handling deal with the failed launch. + run_future = self._run_future + if run_future is None: # pragma: no cover - launch failed after _process was set + return ScriptRunnerState.READY + if run_future.done(): if self._canceled and self._notify_canceled_action_as_failed: return ScriptRunnerState.FAILED if self._canceled and self._runtime_limit_reached: @@ -300,9 +757,29 @@ def exit_code(self) -> Optional[int]: return None def _run(self, args: Sequence[str], time_limit: Optional[timedelta] = None) -> None: + # R4-5 fix: Track launch failure outside the lock, dispatch after. + # _fail_action invokes the user callback synchronously, and a callback + # that calls cancel() would deadlock on self._lock (since + # _cancel_with_resolved_method acquires it unconditionally). No callback + # may run under self._lock — it is a plain Lock, not an RLock. + launch_failure: Optional[str] = None with self._lock: - if self.state != ScriptRunnerState.READY: + # The single-use guard is latched on `_launched`, NOT derived from + # `state`. Deriving it from `state` made the guard depend on every + # future change to that property's classification: when the R5-6 + # change made `state` report READY for a launch that had failed after + # `_process` was assigned, this guard silently opened and a second + # subprocess was launched over the first, orphaning it. A + # "has a launch been attempted" latch cannot drift that way. + # + # `state` is still consulted, because it carries the `_state_override` + # that `_fail_action` sets -- a runner whose setup already failed is + # not READY either. + if self._launched or self.state != ScriptRunnerState.READY: raise RuntimeError("This cannot be used to run a second subprocess.") + # Set before anything that can fail: a runner that has *attempted* a + # launch must never be reusable, whether or not the attempt worked. + self._launched = True if is_posix(): script = self._generate_command_shell_script(args) filehandle, filename = mkstemp( @@ -329,35 +806,55 @@ def _run(self, args: Sequence[str], time_limit: Optional[timedelta] = None) -> N args, self._user, self._os_env_vars, str(self._session_working_directory) ) except RuntimeError as e: - # Make use of the action filter to surface the failure reason to - # the customer. - self._logger.info( - f"openjd_fail: {str(e)}", - extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), - ) - self._state_override = ScriptRunnerState.FAILED - # We haven't started the future yet that runs the process, - # but the Session still needs to know that the action is over. - if self._callback is not None: - self._callback(ActionState.FAILED) - return + # Record failure, don't dispatch yet — we're under the lock. + launch_failure = str(e) + + # Only proceed with subprocess creation if no launch failure occurred + if launch_failure is None: + subprocess_args = [filename] if is_posix() else args + process = LoggingSubprocess( + logger=self._logger, + args=subprocess_args, + user=self._user, + os_env_vars=self._os_env_vars, + working_dir=str(self._session_working_directory), + ) - subprocess_args = [filename] if is_posix() else args - self._process = LoggingSubprocess( - logger=self._logger, - args=subprocess_args, - user=self._user, - os_env_vars=self._os_env_vars, - working_dir=str(self._session_working_directory), - ) + if time_limit: + self._runtime_limit = Timer(time_limit.total_seconds(), self._on_timelimit) + self._runtime_limit.start() + + if self._print_section_banner: + log_subsection_banner(self._logger, "Phase: Running action") + run_future = self._pool.submit(process.run) + + # Publish the pair together, and only once both exist. `state` is + # read without the lock, so assigning `self._process` before the + # future existed left a window -- on the ordinary success path, + # not only on failure -- in which a polling consumer observed a + # process with no future. That combination is what the R5-6 change + # then had to classify, and misclassifying it is what opened the + # guard above. Removing the window removes the question. + self._process = process + self._run_future = run_future + + # Dispatch launch failure outside the lock + if launch_failure is not None: + self._fail_action(launch_failure) + return - if time_limit: - self._runtime_limit = Timer(time_limit.total_seconds(), self._on_timelimit) - self._runtime_limit.start() + # At this point, launch succeeded so _process and _run_future are set. + # R5-6: explicit raises, not asserts. The R4-5 restructuring above depends + # on these two being set together whenever `launch_failure is None`, and + # under `python -O` an `assert` here would strip that check and let the + # method carry on to `add_done_callback`/`wait_until_started` on None -- + # silently reverting the fix this block exists to implement. + if self._run_future is None or self._process is None: # pragma: no cover - defensive + raise RuntimeError( + "Internal error: the action's subprocess was not created despite a successful " + "launch." + ) - if self._print_section_banner: - log_subsection_banner(self._logger, "Phase: Running action") - self._run_future = self._pool.submit(self._process.run) # Intentionally leave the lock section. If the process was *really* fast, # then it's possible for the future to have finished before we get to add # the done-callback. That results in the done-callback being called from @@ -371,24 +868,72 @@ def _run(self, args: Sequence[str], time_limit: Optional[timedelta] = None) -> N # before we check self.state self._process.wait_until_started() + # A cancel that landed during setup, before there was a running + # subprocess to signal, is applied now rather than dropped. Read under + # the lock so the handoff is serialized against the writer in + # _cancel_with_resolved_method; _cancel takes the lock itself, so it is + # called outside. + with self._lock: + pending = self._pending_cancel + self._pending_cancel = None + if pending is not None and self._resolved_cancel_method is not None: + self._cancel(self._resolved_cancel_method, *pending) + if self.state == ScriptRunnerState.RUNNING and self._callback is not None: # Let the caller know that the process is running. self._callback(ActionState.RUNNING) def _generate_command_shell_script(self, args: Sequence[str]) -> str: - """Generate a shell script for running a command given by the args.""" + """Generate a shell script for running a command given by the args. + + Everything interpolated into this script is quoted or validated, because + the result is ``exec``'d by ``/bin/sh``: a single unquoted metacharacter + anywhere in it is arbitrary code execution as the session user. + """ script = list[str]() script.append("#!/bin/sh\n") if self._os_env_vars: for name, value in self._os_env_vars.items(): + if not POSIX_SHELL_NAME_RE.fullmatch(name): + # R5-5 fix: the value was already quoted, but the NAME was + # interpolated raw -- so a name containing ';', '$(' or a + # newline injected commands here. + # + # Skipped rather than escaped or rejected. A shell variable + # name cannot legally contain a metacharacter, so there is + # nothing to escape *to*: `export 'a;b'=v` is not a valid + # assignment, and `export ProgramFiles(x86)=v` -- a real + # Windows variable name -- is an outright /bin/sh syntax + # error that would fail the whole action. Skipping is + # strictly better than both: the variable still reaches the + # subprocess through Popen's `env=`, which does not go + # through a shell, so nothing that used to work stops + # working. + # The warning does not promise the variable still arrives. + # For a same-user action it does, via Popen's `env=`. For a + # cross-user action it does not: the subprocess is launched + # through `sudo -u -i`, and `-i` starts a login shell + # that resets the environment, so this script's `export` lines + # are the only channel that reaches the workload. + self._logger.warning( + f"Not exporting environment variable {name!r} from the action's shell " + "script: the name is not a valid POSIX shell identifier, which /bin/sh " + "cannot express. For an action run as another user this means the " + "variable will not reach the subprocess at all.", + extra=LogExtraInfo(openjd_log_content=LogContent.PARAMETER_INFO), + ) + continue if value is None: script.append(f"unset {name}") else: script.append(f"export {name}={shlex.quote(value)}") if self._startup_directory is not None: - # Note: Single quotes around the path as it may have spaces, and we don't want to - # process any shell commands in the path. - script.append(f"cd '{self._startup_directory}'") + # R5-4 fix: shlex.quote, not hand-written single quotes. A single + # quote *in the path* closed the quoted region and everything after + # it was interpreted by /bin/sh. Note the env var value two lines + # above has always used shlex.quote -- the safe idiom was already + # imported and in use in this same function. + script.append(f"cd {shlex.quote(str(self._startup_directory))}") script.append("exec " + shlex.join(args)) return "\n".join(script) @@ -398,10 +943,35 @@ def _materialize_files( files: EmbeddedFilesListType, dest_directory: Path, symtab: SymbolTable, + let_bindings: Optional[list[str]] = None, + preallocated_records: Optional[list[_FileRecord]] = None, ) -> None: """Helper for derived classes that wraps all of the logic around materializing embedded files to disk. + When ``let_bindings`` is given, they are evaluated between file-path + allocation and content writing (RFC 0005, mirroring the openjd-rs + runners): a file's *path* never depends on ``let`` values (filenames + are plain strings), so ``Env.File.*``/``Task.File.*`` are available to + the bindings, while a file's ``data`` is written afterwards so it can + reference let-bound values. + + When ``preallocated_records`` is given (RFC 0008: a wrap + environment's files, whose paths the Session allocates once and + reuses across wrap-hook invocations), path allocation is skipped; + the records' symbols are defined in ``symtab`` and the contents are + re-resolved and written as usual. + + Note on the per-invocation rewrite: the resolved bytes are in fact + invariant across a wrap environment's hook invocations, because model + validation rejects ``WrappedAction.*`` both in an environment script's + ``data`` and in its ``let`` bindings, and every other symbol a wrap + env's ``data`` may reference (``Param.*``, ``Session.*``, ``Job.Name``, + the now-stable ``Env.File.*``) is fixed for the session. The rewrite is + kept anyway, deliberately: it costs one small write immediately before + spawning a subprocess — the same cost class as the spawn — and it makes + each invocation deterministic even if a previous invocation's subprocess + modified a ``runnable`` embedded script in place. """ file_writer = EmbeddedFiles( logger=self._logger, @@ -410,21 +980,30 @@ def _materialize_files( user=self._user, ) try: - file_writer.materialize(files, symtab) - except RuntimeError as exc: - # Had a problem writing at least one file to disk. - # Surface the error. - # Make use of the action filter to surface the failure reason to - # the customer. - self._logger.info( - f"openjd_fail: {str(exc)}", - extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), - ) - self._state_override = ScriptRunnerState.FAILED - # We haven't started the future yet that runs the process, - # but the Session still needs to know that the action is over. - if self._callback is not None: - self._callback(ActionState.FAILED) + if preallocated_records is not None: + records = preallocated_records + file_writer.register_file_paths(records, symtab) + else: + records = file_writer.allocate_file_paths(files, symtab) + if let_bindings: + apply_let_bindings(symtab=symtab, let_bindings=let_bindings) + file_writer.write_file_contents(records, symtab) + except (RuntimeError, ValueError) as exc: + # Had a problem writing at least one file to disk, or evaluating + # a `let` binding (FormatStringError/ExpressionError subclass + # ValueError). Surface the error. + self._fail_action(str(exc)) + + def _apply_let_bindings_or_fail(self, symtab: SymbolTable, let_bindings: list[str]) -> bool: + """Evaluate the script's EXPR ``let`` bindings into ``symtab``. On an + evaluation error the action is failed through the normal failure path + (openjd_fail log, FAILED state, callback). Returns True on success.""" + try: + apply_let_bindings(symtab=symtab, let_bindings=let_bindings) + except ValueError as exc: + self._fail_action(str(exc)) + return False + return True def _run_action( self, @@ -432,6 +1011,7 @@ def _run_action( symtab: SymbolTable, *, default_timeout: Optional[timedelta] = None, + default_notify_period_seconds: int = 30, ) -> None: """Helper for derived classes to run a specific Action. @@ -443,45 +1023,189 @@ def _run_action( default_timeout (Optional[timedelta], optional): Default timeout duration for the action if no timeout is specified in the action. The default behaviour if None is passed will allow the action to run indefinitely until it completes. + default_notify_period_seconds (int): The Template Schemas 5.3.2 + positional default applied when a NOTIFY_THEN_TERMINATE + cancelation omits its notify period (120 for a task's onRun, + 30 for any other action). """ assert isinstance(action, Action_2023_09) try: command = [action.command.resolve(symtab=symtab)] - if action.args is not None: - command.extend(s.resolve(symtab=symtab) for s in action.args) + # RFC 0005 §1.3.2 typed argument semantics (null skip, list + # flattening) — see resolve_action_arg_values, shared with the + # RFC 0008 WrappedAction.Args injection. + command.extend(resolve_action_arg_values(action.args, symtab)) except FormatStringError as exc: # Extremely unlikely since a JobTemplate needs to have passed # validation before we could be running it, but just to be safe. - self._logger.info( - f"openjd_fail: {str(exc)}", - extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), - ) - self._state_override = ScriptRunnerState.FAILED - # We haven't started the future yet that runs the process, - # but the Session still needs to know that the action is over. - if self._callback is not None: - self._callback(ActionState.FAILED) + self._fail_action(str(exc)) else: time_limit: Optional[timedelta] = default_timeout - if action.timeout: - time_limit = timedelta(seconds=action.timeout) # type: ignore[arg-type] + # A FormatString timeout (FEATURE_BUNDLE_1) is resolved right + # before the action runs. A whole-field expression that + # resolves to a typed null — e.g. forwarding + # `timeout: "{{WrappedAction.Timeout}}"` (RFC 0008) when the + # wrapped action specified no timeout — is treated as if the + # field were not provided, so the positional default applies. + try: + seconds = resolve_optional_int_field( + action.timeout, symtab, ge=1, description="timeout" + ) + except ValueError as exc: + # FormatStringError (resolution failure) subclasses + # ValueError, so this covers both a failed resolution and a + # non-positive-integer resolved value. + self._fail_action(str(exc)) + return + if seconds is not None: + # Assigned unconditionally: a declared timeout always replaces + # `default_timeout`, including when it is too large to enforce + # (in which case the action runs unbounded — see + # _timeout_from_seconds). Assigning only in the enforceable + # case would silently downgrade an oversized timeout to the + # positional default, e.g. an environment exit's 5 minutes. + time_limit = _timeout_from_seconds(seconds, self._logger) + # Resolve the action's effective cancelation NOW, against the + # same final scope the command/args/timeout resolved with — a + # deferred (format-string) mode or FEATURE_BUNDLE_1 notify + # period may reference script-level `let` bindings or + # Env.File.*/Task.File.* symbols that only exist in this scope. + # This matches the openjd-rs runtime (run_action resolves + # cancel_method_for_action before launching): an unresolvable or + # invalid cancelation fails the action at start instead of + # surfacing only if a cancel later occurs. The runners' + # cancel() consumes the stored method. + try: + mode, period = resolve_effective_cancelation(action.cancelation, symtab) + except ValueError as exc: + # FormatStringError (resolution failure) subclasses ValueError. + self._fail_action(str(exc)) + return + if mode != CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value: + # Note: The default cancelation for a 2023-09 script is Terminate + self._resolved_cancel_method = TerminateCancelMethod() + else: + self._resolved_cancel_method = NotifyCancelMethod( + terminate_delay=timedelta( + seconds=(period if period is not None else default_notify_period_seconds) + ) + ) self._run(command, time_limit) + def _cancel_with_resolved_method( + self, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False + ) -> None: + """Shared implementation of the runners' :meth:`cancel`: cancel with + the effective cancel method that :meth:`_run_action` resolved at + launch time. + + A cancel that arrives before the subprocess is running is remembered and + applied by :meth:`_run` as soon as it starts (see :attr:`_pending_cancel`) + — `cancel()` is called from another thread, so "not running yet" is a + race, not a no-op. Only when no action will ever be launched (setup + failed, or there was no action to run) is there genuinely nothing to + cancel. + """ + with self._lock: + # Decide-and-record must be atomic with respect to _run creating and + # starting the subprocess, otherwise a cancel can land between the + # two and be dropped by both sides: this method would see a process + # and hand off to _cancel, which has nothing to signal yet, while + # _run has already passed the point where it consumes a pending + # cancel. Keyed on has_started rather than on the object existing, + # for the same reason. + if self._state_override is not None: + # Terminal before launch: setup failed, or there was no action. + return + process = self._process + if process is None or not process.has_started: + # F3 fix: Monotonic merge for duplicate pending cancels. If a + # cancel is already pending, merge the new request: take the + # minimum time_limit (tighter deadline wins, treating None as + # unlimited), and OR the mark_action_failed flags (once failed, + # always failed). + if self._pending_cancel is not None: + prev_limit, prev_failed = self._pending_cancel + # Merge time limits: None means unlimited, so a defined limit beats None + if time_limit is None: + merged_limit = prev_limit + elif prev_limit is None: + merged_limit = time_limit + else: + merged_limit = min(time_limit, prev_limit) + merged_failed = mark_action_failed or prev_failed + self._pending_cancel = (merged_limit, merged_failed) + else: + self._pending_cancel = (time_limit, mark_action_failed) + return + method = self._resolved_cancel_method + if method is None: # pragma: no cover - defensive + return + # Note: If the given time_limit is less than that in the method, then the time_limit will be what's used. + # Called outside the lock: _cancel takes it itself. + self._cancel(method, time_limit, mark_action_failed) + + def _signal_process( + self, process: LoggingSubprocess, method: Literal["terminate", "notify"] + ) -> None: + """Send ``method`` to ``process``, warning rather than raising if it fails. + + ``process`` is a parameter rather than a read of ``self._process`` so the + notify-period timer can pass the lock-free local snapshot it already + holds, instead of re-reading an attribute another thread may have + cleared. + + An ``OSError`` here is close to impossible -- if we could start the + process we can signal it -- but a cancel path is the wrong place to + raise: the caller is already unwinding a cancel, and losing the + surrounding bookkeeping to a signal failure is worse than a warning. + """ + try: + if method == "notify": + process.notify() + else: + process.terminate() + except OSError as err: # pragma: nocover + self._logger.warning( + f"Cancelation could not send {method} signal to process {process.pid}: {str(err)}", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO + ), + ) + def _cancel( self, method: CancelMethod, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False, ) -> None: - # For the type checkers - assert self._process is not None - # Nothing to do if it's not running. - if not self._process.is_running: + if self._process is None: + # A cancel that raced action setup: nothing to signal yet. Callers + # go through _cancel_with_resolved_method, which records it as a + # pending cancel instead — an early return here rather than an + # assert, so no bare AssertionError can reach the public API. return with self._lock: + # F4 fix: Check liveness under the lock. Without this, a completion + # racing a cancel/timeout could see is_running=True outside the lock, + # enter here, and then find is_running=False (or worse, still True + # but the callback has already fired) — no linearization point. + # Moving the check inside the lock lets _on_process_exit's clearing + # of _pending_cancel act as the arbiter: if the process exited, any + # pending cancel was already consumed there. + if not self._process.is_running: + return + self._canceled = True - self._notify_canceled_action_as_failed = mark_action_failed + # R4-G7 fix (F3 live path): Monotonic merge for failure attribution. + # If a previous cancel already set mark_action_failed=True (e.g., a + # parse failure triggered cancel), a subsequent timeout or manual + # cancel must not erase that determination. The action should report + # FAILED, not TIMEOUT/CANCELED. + self._notify_canceled_action_as_failed = ( + self._notify_canceled_action_as_failed or mark_action_failed + ) now = datetime.now(timezone.utc) now_str = now.strftime(TIME_FORMAT_STR) if self._cancel_gracetime_timer is not None: @@ -495,17 +1219,7 @@ def _cancel( f"Canceling subprocess {str(self._process.pid)} via termination method at {now_str}.", extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), ) - try: - self._process.terminate() - except OSError as err: # pragma: nocover - # Being paranoid. Won't happen... if we could start the process, then we can send it a signal - self._logger.warning( - f"Cancelation could not send terminate signal to process {self._process.pid}: {str(err)}", - extra=LogExtraInfo( - openjd_log_content=LogContent.PROCESS_CONTROL - | LogContent.EXCEPTION_INFO - ), - ) + self._signal_process(self._process, "terminate") else: self._logger.info( f"Canceling subprocess {str(self._process.pid)} via notify then terminate method at {now_str}.", @@ -533,26 +1247,39 @@ def _cancel( # when we'll send the SIGKILL) grace_end_time_str = self._cancel_gracetime_end.strftime(TIME_FORMAT_STR) notify_end = json.dumps({"NotifyEnd": grace_end_time_str}) - write_file_for_user( - self._session_working_directory / "cancel_info.json", notify_end, self._user - ) - self._logger.info( - f"Grace period ends at {grace_end_time_str}", - extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), - ) - - # 2) Send the notify try: - self._process.notify() - except OSError as err: # pragma: nocover - # Being paranoid. Won't happen... if we could start the process, then we can send it a signal + write_file_for_user( + self._session_working_directory / "cancel_info.json", notify_end, self._user + ) + except OSError as err: + # F6 fix: If we cannot write the cancel_info.json (disk full, permission + # denied, etc.), log and fall back to immediate termination. A script + # waiting on that file would hang forever otherwise. self._logger.warning( - f"Cancelation could not send notify signal to process {self._process.pid}: {str(err)}", + f"Failed to write cancel_info.json: {err}. Falling back to immediate termination.", extra=LogExtraInfo( openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO ), ) + try: + self._process.terminate() + except OSError as term_err: # pragma: nocover + self._logger.warning( + f"Fallback termination also failed: {term_err}", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL + | LogContent.EXCEPTION_INFO + ), + ) + return + self._logger.info( + f"Grace period ends at {grace_end_time_str}", + extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), + ) + + # 2) Send the notify + self._signal_process(self._process, "notify") # 4) Set up the timer to send the terminate signal self._cancel_gracetime_timer = Timer( @@ -562,8 +1289,22 @@ def _cancel( def _on_process_exit(self, future: Future) -> None: """This is invoked as a callback when run_future is done.""" - assert self._run_future is not None + # R5-6: read the exception from the future the completion actually fired + # for. This runs on the pool worker (or, for a very fast process, on the + # launching thread), so an AssertionError on `self._run_future` would only + # surface through threading.excepthook -- and under `python -O` the assert + # is stripped and the `.exception()` read below becomes an AttributeError + # in the same unobservable place. `future` is always the right object; + # `self._run_future` can additionally be cleared by cleanup(). + run_future = future with self._lock: + # F2 fix: Claim _pending_cancel atomically before signalling completion. + # A cancel racing completion would otherwise see the process still + # "running" (in _cancel_with_resolved_method) and hand off to _cancel, + # which then finds is_running=False and no-ops. By consuming the + # pending here we prevent that lost-cancel window. + self._pending_cancel = None + if self._runtime_limit is not None: self._runtime_limit.cancel() self._runtime_limit = None @@ -572,7 +1313,7 @@ def _on_process_exit(self, future: Future) -> None: self._cancel_gracetime_timer.cancel() self._cancel_gracetime_timer = None - if exc := self._run_future.exception(): + if exc := run_future.exception(): self._logger.error( f"Error running subprocess: {str(exc)}", extra=LogExtraInfo( @@ -580,31 +1321,70 @@ def _on_process_exit(self, future: Future) -> None: ), ) - if self._callback is not None: - self._callback(ActionState(self.state.value)) + # F8 fix: Invoke callback outside the lock and wrap in try/except. An + # observer exception must not prevent the child process from being + # reaped or cause resource leaks. The callback is invoked outside the + # lock since it may be slow and shouldn't block other operations. + if self._callback is not None: + try: + self._callback(self._terminal_action_state()) + except Exception as exc: + self._logger.error( + f"Exception in action callback: {exc}", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO + ), + ) + + def _terminal_action_state(self) -> ActionState: + """The :class:`ActionState` to publish now that the action is over. + + ``ActionState`` has no member for ``ScriptRunnerState.READY`` or + ``CANCELING``, so ``ActionState(self.state.value)`` raises ``ValueError`` + for either. That mattered: the F8 ``try/except`` around the callback + catches the ValueError and logs it, so the consumer receives **no + terminal callback at all** and waits forever on an action that has + finished. Reached when the runner's state cannot be classified -- e.g. + the process/future pair left inconsistent by a failed launch. + + The action is over by the time we are called, so an unclassifiable state + is reported as FAILED. Publishing a wrong-but-terminal state is strictly + better than publishing nothing. + """ + state = self.state + try: + return ActionState(state.value) + except ValueError: + self._logger.error( + f"Action completed in the non-terminal state '{state.value}'; reporting it as " + "FAILED. This indicates an internal error in this runtime.", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO + ), + ) + return ActionState.FAILED def _on_notify_period_end(self) -> None: """This is invoked when the grace period in a NOTIFY_THEN_TERMINATE cancelation has expired. """ - assert self._process is not None with self._lock: self._cancel_gracetime_timer = None + process = self._process + # R5-6: a plain check, not `assert`. This runs on a threading.Timer + # thread, so any exception raised here reaches only + # threading.excepthook -- the grace period would appear to have silently + # done nothing. There is nothing left to terminate if the process is + # already gone, which is the ordinary outcome when the action completed + # during the grace period. + if process is None: + return self._logger.info( "Notify period ended. Terminate at %s", datetime.now(timezone.utc).strftime(TIME_FORMAT_STR), extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), ) - try: - self._process.terminate() - except OSError as err: # pragma: nocover - # Being paranoid. Won't happen... if we could start the process, then we can send it a kill signal - self._logger.warning( - f"Cancelation could not send terminate signal to process {self._process.pid}: {str(err)}", - extra=LogExtraInfo( - openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO - ), - ) + self._signal_process(process, "terminate") def _on_timelimit(self) -> None: """Callback that is invoked when the runtime limit of the running diff --git a/src/openjd/sessions/_runner_env_script.py b/src/openjd/sessions/_runner_env_script.py index 2ce1962c..235405a1 100644 --- a/src/openjd/sessions/_runner_env_script.py +++ b/src/openjd/sessions/_runner_env_script.py @@ -6,25 +6,28 @@ from typing import Callable, Optional from openjd.model import SymbolTable -from openjd.model.v2023_09 import Action as Action_2023_09 -from openjd.model.v2023_09 import CancelationMode as CancelationMode_2023_09 -from openjd.model.v2023_09 import ( - CancelationMethodNotifyThenTerminate as CancelationMethodNotifyThenTerminate_2023_09, -) from openjd.model.v2023_09 import EnvironmentScript as EnvironmentScript_2023_09 -from ._embedded_files import EmbeddedFilesScope +from ._embedded_files import EmbeddedFilesScope, _FileRecord from ._logging import log_subsection_banner from ._runner_base import ( - CancelMethod, - NotifyCancelMethod, ScriptRunnerBase, ScriptRunnerState, - TerminateCancelMethod, ) from ._session_user import SessionUser -from ._types import ActionModel, ActionState, EnvironmentScriptModel +from ._types import ( + ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS, + ActionModel, + ActionState, + EnvironmentScriptModel, +) + +__all__ = ("EnvironmentScriptRunner", "WRAP_HOOK_ACTION_NAMES") -__all__ = ("EnvironmentScriptRunner",) + +WRAP_HOOK_ACTION_NAMES = ("onWrapEnvEnter", "onWrapTaskRun", "onWrapEnvExit") +"""The three RFC 0008 wrap-hook action names an Environment's script may +define. Single-sourced here for the runner's hook dispatch and the Session's +wrap-environment lookup/validation.""" _ENV_EXIT_DEFAULT_TIMEOUT = timedelta(minutes=5) @@ -55,6 +58,13 @@ class EnvironmentScriptRunner(ScriptRunnerBase): """If defined, then this is the action that is currently running, or was last run. """ + _preallocated_file_records: Optional[list[_FileRecord]] + """RFC 0008: the script's embedded-file records with on-disk paths already + allocated by the Session (a wrap environment's files are allocated once + and reused across wrap-hook invocations). When set, the runner skips path + allocation and only re-resolves/rewrites the file contents. + """ + def __init__( self, *, @@ -72,6 +82,9 @@ def __init__( symtab: SymbolTable, # Directory within which files/attachments should be materialized session_files_directory: Path, + # RFC 0008: pre-allocated embedded-file records to reuse (see + # _preallocated_file_records) + preallocated_file_records: Optional[list[_FileRecord]] = None, ): """ Arguments (from base class): @@ -91,6 +104,11 @@ def __init__( Script's scope (exluding any symbols defined within the Step Script itself). session_files_directory (Path): The location in the filesystem where embedded files will be materialized. + preallocated_file_records (Optional[list[_FileRecord]]): RFC 0008: the script's + embedded-file records with on-disk paths already allocated by the Session + (a wrap environment's files are allocated once and reused across wrap-hook + invocations). When given, the runner skips path allocation and only + re-resolves/rewrites the file contents. """ super().__init__( logger=logger, @@ -104,6 +122,7 @@ def __init__( self._symtab = symtab self._session_files_directory = session_files_directory self._action = None + self._preallocated_file_records = preallocated_file_records if self._environment_script and not isinstance( self._environment_script, EnvironmentScript_2023_09 @@ -120,7 +139,13 @@ def _run_env_action( log_subsection_banner(self._logger, "Phase: Setup") - # Write any embedded files to disk + let_bindings = ( + self._environment_script.let if self._environment_script is not None else None + ) + # Write any embedded files to disk. File paths are allocated before + # the script's EXPR `let` bindings evaluate (so bindings can reference + # Env.File.*), and contents are written after (so `data` can reference + # let-bound values) — mirroring the openjd-rs runner. if ( self._environment_script is not None and self._environment_script.embeddedFiles is not None @@ -132,15 +157,26 @@ def _run_env_action( self._environment_script.embeddedFiles, self._session_files_directory, symtab, + let_bindings=let_bindings, + preallocated_records=self._preallocated_file_records, ) if self.state == ScriptRunnerState.FAILED: return + elif let_bindings: + symtab = SymbolTable(source=self._symtab) + if not self._apply_let_bindings_or_fail(symtab, let_bindings): + return else: symtab = self._symtab # Construct the command by evalutating the format strings in the command self._action = action - self._run_action(self._action, symtab, default_timeout=default_timeout) + self._run_action( + self._action, + symtab, + default_timeout=default_timeout, + default_notify_period_seconds=ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS, + ) def enter(self) -> None: """Run the Environment's onEnter action.""" @@ -181,34 +217,69 @@ def exit(self) -> None: default_timeout=_ENV_EXIT_DEFAULT_TIMEOUT, ) - def cancel( - self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False - ) -> None: - if self._action is None: - # Nothing to do. - return + def wrap_task_run(self) -> None: + """Run the Environment's onWrapTaskRun action, wrapping a task's onRun.""" + self._run_wrap_hook("onWrapTaskRun") + + def wrap_env_enter(self) -> None: + """RFC 0008: run this Environment's ``onWrapEnvEnter`` action, + substituting it for an inner environment's ``onEnter``.""" + self._run_wrap_hook("onWrapEnvEnter") + + def wrap_env_exit(self) -> None: + """RFC 0008: run this Environment's ``onWrapEnvExit`` action, + substituting it for an inner environment's ``onExit``.""" + self._run_wrap_hook("onWrapEnvExit", default_timeout=_ENV_EXIT_DEFAULT_TIMEOUT) + + def _run_wrap_hook(self, hook: str, *, default_timeout: Optional[timedelta] = None) -> None: + """Common dispatch for the three RFC 0008 wrap hooks. ``hook`` is + one of ``onWrapEnvEnter``, ``onWrapTaskRun``, or ``onWrapEnvExit``.""" + if hook not in WRAP_HOOK_ACTION_NAMES: + # Distinguish a caller typo from model skew: the hasattr check + # below would otherwise report a mistyped hook name as an + # incompatible openjd-model. + raise ValueError(f"Unknown wrap hook name: {hook}") + if self.state != ScriptRunnerState.READY: + raise RuntimeError("This cannot be used to run a second subprocess.") # For the type checker - assert isinstance(self._action, Action_2023_09) + if self._environment_script is not None: + assert isinstance(self._environment_script, EnvironmentScript_2023_09) - method: CancelMethod - if ( - self._action.cancelation is None - or self._action.cancelation.mode == CancelationMode_2023_09.TERMINATE - ): - # Note: Default cancelation for a 2023-09 Step Script is Terminate - method = TerminateCancelMethod() - else: - model_cancel_method = self._action.cancelation - # For the type checker - assert isinstance(model_cancel_method, CancelationMethodNotifyThenTerminate_2023_09) - if model_cancel_method.notifyPeriodInSeconds is None: - # Default grace period is 30s for a 2023-09 Environment Script's notify cancel - method = NotifyCancelMethod(terminate_delay=timedelta(seconds=30)) - else: - method = NotifyCancelMethod( - terminate_delay=timedelta(seconds=model_cancel_method.notifyPeriodInSeconds) # type: ignore[arg-type] + action = None + if self._environment_script is not None: + actions = self._environment_script.actions + # Distinguish "the model has no such field" from "the hook is not + # declared". Only the latter is a legitimate no-op; the former means + # the model's field was renamed, and a getattr default would silently + # report SUCCESS *without running the hook*. + if not hasattr(actions, hook): + raise RuntimeError( + f"This Environment's actions do not support the wrap hook '{hook}'. " + "The installed openjd-model may be incompatible with this runtime." ) + action = getattr(actions, hook) + if action is None: + self._state_override = ScriptRunnerState.SUCCESS + # Nothing to do, no wrap action defined. Call the callback + # to inform the caller that the run is complete, and then exit. + if self._callback is not None: + self._callback(ActionState.SUCCESS) + return - # Note: If the given time_limit is less than that in the method, then the time_limit will be what's used. - self._cancel(method, time_limit, mark_action_failed) + self._run_env_action(action, default_timeout=default_timeout) + + def cancel( + self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False + ) -> None: + # Always route through the base class handoff, even before _action is + # assigned. A cancel landing during environment-action setup (embedded- + # file writes, let evaluation) must be recorded in _pending_cancel and + # applied when the subprocess launches — F1 fix: the old `_action is + # None` guard returned early without entering the pending-cancel path, + # losing the cancel entirely during the setup window. + # + # _cancel_with_resolved_method handles the pre-launch case: it records + # the request in _pending_cancel when the subprocess hasn't started, + # and _run applies it once the child exists. + self._cancel_with_resolved_method(time_limit, mark_action_failed) diff --git a/src/openjd/sessions/_runner_step_script.py b/src/openjd/sessions/_runner_step_script.py index a4d2e654..1d49736b 100644 --- a/src/openjd/sessions/_runner_step_script.py +++ b/src/openjd/sessions/_runner_step_script.py @@ -6,22 +6,15 @@ from typing import Callable, Optional from openjd.model import SymbolTable -from openjd.model.v2023_09 import CancelationMode as CancelationMode_2023_09 from openjd.model.v2023_09 import StepScript as StepScript_2023_09 -from openjd.model.v2023_09 import ( - CancelationMethodNotifyThenTerminate as CancelationMethodNotifyThenTerminate_2023_09, -) from ._embedded_files import EmbeddedFilesScope from ._logging import log_subsection_banner from ._runner_base import ( - CancelMethod, - NotifyCancelMethod, ScriptRunnerBase, ScriptRunnerState, - TerminateCancelMethod, ) from ._session_user import SessionUser -from ._types import ActionState, StepScriptModel +from ._types import TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS, ActionState, StepScriptModel __all__ = ("StepScriptRunner",) @@ -104,7 +97,11 @@ def run(self) -> None: # For the type checker. assert isinstance(self._script, StepScript_2023_09) - # Write any embedded files to disk + let_bindings = self._script.let + # Write any embedded files to disk. File paths are allocated before + # the script's EXPR `let` bindings evaluate (so bindings can reference + # Task.File.*), and contents are written after (so `data` can + # reference let-bound values) — mirroring the openjd-rs runner. if self._script.embeddedFiles is not None: symtab = SymbolTable(source=self._symtab) self._materialize_files( @@ -112,39 +109,28 @@ def run(self) -> None: self._script.embeddedFiles, self._session_files_directory, symtab, + let_bindings=let_bindings, ) if self.state == ScriptRunnerState.FAILED: return + elif let_bindings: + symtab = SymbolTable(source=self._symtab) + if not self._apply_let_bindings_or_fail(symtab, let_bindings): + return else: symtab = self._symtab # Construct the command by evalutating the format strings in the command - self._run_action(self._script.actions.onRun, symtab) + self._run_action( + self._script.actions.onRun, + symtab, + default_notify_period_seconds=TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS, + ) def cancel( self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False ) -> None: - # For the type checker. - assert isinstance(self._script, StepScript_2023_09) - - method: CancelMethod - if ( - self._script.actions.onRun.cancelation is None - or self._script.actions.onRun.cancelation.mode == CancelationMode_2023_09.TERMINATE - ): - # Note: Default cancelation for a 2023-09 Step Script is Terminate - method = TerminateCancelMethod() - else: - model_cancel_method = self._script.actions.onRun.cancelation - # For the type checker - assert isinstance(model_cancel_method, CancelationMethodNotifyThenTerminate_2023_09) - if model_cancel_method.notifyPeriodInSeconds is None: - # Default grace period is 120s for a 2023-09 Step Script's notify cancel - method = NotifyCancelMethod(terminate_delay=timedelta(seconds=120)) - else: - method = NotifyCancelMethod( - terminate_delay=timedelta(seconds=model_cancel_method.notifyPeriodInSeconds) # type: ignore[arg-type] - ) - - # Note: If the given time_limit is less than that in the method, then the time_limit will be what's used. - self._cancel(method, time_limit, mark_action_failed) + # Cancel with the effective method resolved at launch time by + # _run_action, against the action's own final scope (its lets and + # Task.File.* symbols) — openjd-rs parity. + self._cancel_with_resolved_method(time_limit, mark_action_failed) diff --git a/src/openjd/sessions/_session.py b/src/openjd/sessions/_session.py index dbb8af49..73063e16 100644 --- a/src/openjd/sessions/_session.py +++ b/src/openjd/sessions/_session.py @@ -14,9 +14,10 @@ from pathlib import Path from tempfile import mkstemp from types import TracebackType -from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union, cast from openjd.model import ( + FormatStringError, JobParameterValues, ParameterValue, ParameterValueType, @@ -37,20 +38,29 @@ ValueReferenceConstants as ValueReferenceConstants_2023_09, ) from ._action_filter import ActionMessageKind, ActionMonitoringFilter -from ._embedded_files import write_file_for_user +from ._embedded_files import EmbeddedFiles, EmbeddedFilesScope, _FileRecord, write_file_for_user from ._logging import LOG, log_section_banner, LoggerAdapter, LogExtraInfo, LogContent from ._os_checker import is_posix, is_windows from ._path_mapping import PathMappingRule -from ._runner_base import ScriptRunnerBase -from ._runner_env_script import EnvironmentScriptRunner +from ._runner_base import ( + ScriptRunnerBase, + apply_let_bindings, + resolve_action_arg_values, + resolve_effective_cancelation, + resolve_optional_int_field, +) +from ._runner_env_script import EnvironmentScriptRunner, WRAP_HOOK_ACTION_NAMES from ._runner_step_script import StepScriptRunner from ._session_user import SessionUser from ._subprocess import LoggingSubprocess from ._tempdir import TempDir, custom_gettempdir from ._types import ( + ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS, + TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS, ActionState, EnvironmentIdentifier, EnvironmentModel, + EnvironmentScriptModel, StepScriptModel, ) from ._version import version @@ -135,6 +145,11 @@ class SimplifiedEnvironmentVariableChanges: """ def __init__(self, initial_variables: Union[dict[str, str], "EnvironmentVariableObject"]): + # Insertion-ordered map of every session-defined variable for this + # environment: the declarative `variables:` map seed plus any + # openjd_env stdout sets/unsets (None = unset). RFC 0008's + # ``WrappedAction.Environment`` surfaces all of these (openjd-rs + # #277); host-inherited variables never enter this map. self._to_set: dict[str, Optional[str]] if is_windows(): @@ -147,13 +162,21 @@ def __init__(self, initial_variables: Union[dict[str, str], "EnvironmentVariable def simplify_ordered_changes(self, changes: list[EnvironmentVariableChange]) -> None: """Apply a given list of sets and unsets to the current state in order""" for change in changes: + name = change.name.upper() if is_windows() else change.name if isinstance(change, EnvironmentVariableSetChange): - self._to_set[change.name.upper() if is_windows() else change.name] = change.value + self._to_set[name] = change.value elif isinstance(change, EnvironmentVariableUnsetChange): - self._to_set[change.name.upper() if is_windows() else change.name] = None + self._to_set[name] = None else: raise ValueError("Unknown type of environment variable change.") + def effective_items(self) -> dict[str, Optional[str]]: + """The effective session-defined variables for this environment, in + insertion order: the declarative ``variables:`` map seed plus any + openjd_env stdout sets/unsets applied so far (``None`` = unset). + Returns a copy; mutating it does not affect this object.""" + return dict(self._to_set) + def apply_to_environment(self, env_vars: dict[str, Optional[str]]) -> None: """Modify a given dictionary of environment variables to reflect the changes""" if is_windows(): @@ -256,6 +279,18 @@ class Session(object): """OS environment variables defined by Open Job Description Environments """ + _wrap_env_file_records: dict[EnvironmentIdentifier, list["_FileRecord"]] + """RFC 0008: per wrap environment, the embedded-file records whose on-disk + paths were allocated on the environment's first wrap-hook invocation and + are reused for every subsequent invocation — so the ``Env.File.*`` symbols + stay stable across tasks and unnamed files do not accumulate on disk. The + file *contents* are still re-resolved and rewritten per invocation by the + runner — not because they can change (validation rejects ``WrappedAction.*`` + in an environment script's ``data`` and ``let``, so they cannot), but so that + every invocation starts from the authored content even if a previous one + modified the file on disk. See ScriptRunnerBase._materialize_files. + """ + _log_filter: Filter """The handler that we've hooked to the LOG. Removed when the Session is deleted. """ @@ -323,6 +358,7 @@ def __init__( *, session_id: str, job_parameter_values: JobParameterValues, + job_name: Optional[str] = None, path_mapping_rules: Optional[list[PathMappingRule]] = None, retain_working_dir: bool = False, user: Optional[SessionUser] = None, @@ -339,6 +375,11 @@ def __init__( job_parameter_values (JobParameterValues): Values for any defined Job Parameters. This is a dictionary where the keys are parameter names, and the values are instances of ParameterValue (a dataclass containing the type and value of the parameter) + job_name (Optional[str]): The resolved name of the Job this Session belongs to. + When provided, it is seeded as the ``Job.Name`` template variable + (RFC 0005; Template Schemas §7.3.1, EXPR extension) for the session's actions — + mirroring the Job.Name symbol that openjd-rs carries in its + session symbol tables. Defaults to None (no Job.Name symbol). path_mapping_rules (Optional[list[PathMappingRule]]): A list of the path mapping rules to apply within all actions running within this session. Defaults to None. retain_working_dir (bool, optional): If set, then the Session's Working Directory @@ -382,21 +423,39 @@ def __init__( self._session_id = session_id self._ending_only = False self._environments = dict() + # Extra EXPR `let` bindings supplied when an environment was entered + # (e.g. the owning step's step-level bindings), re-applied when the + # environment exits so its onExit resolves in the same scope. + self._environment_extra_let_bindings: dict[EnvironmentIdentifier, list[str]] = dict() + # The owning step's name supplied when an environment was entered + # (seeds Step.Name, RFC 0005 EXPR), re-seeded when the environment + # exits so the re-applied extra `let` bindings resolve in the same + # scope. + self._environment_step_names: dict[EnvironmentIdentifier, str] = dict() self._environments_entered = list() self._runner = None self._running_environment_identifier = None self._process_env = dict(os_env_vars) if os_env_vars else dict() self._created_env_vars = dict() + self._wrap_env_file_records = dict() self._retain_working_dir = retain_working_dir self._user = user self._job_parameter_values = dict(job_parameter_values) if job_parameter_values else dict() + self._job_name = job_name self._cleanup_called = False self._callback = callback self._path_mapping_rules = path_mapping_rules[:] if path_mapping_rules else None if self._path_mapping_rules is not None: # Path mapping rules are applied in order of longest to shortest source path, # so sort them for when we apply them. - self._path_mapping_rules.sort(key=lambda rule: -len(rule.source_path.parts)) + self._path_mapping_rules.sort(key=lambda rule: -rule._source_path_component_count()) + # Engine (openjd.expr) form of the path mapping rules, seeded into the + # session's symbol tables so EXPR host-context functions such as + # apply_path_mapping() apply the session's rules at run time — + # mirroring openjd-rs's session-scope HostContext::WithRules. An empty + # list still constructs a host context (the session IS host scope); + # the rules are simply empty. + self._expr_host_rules = self._build_expr_host_rules() self._session_root_directory = session_root_directory if self._session_root_directory is not None: if not self._session_root_directory.is_dir(): @@ -538,8 +597,19 @@ def working_directory(self) -> Path: """The directory that was created for this Session's working files. This is available in a Job Template's format string expressions as Session.WorkingDirectory + + Raises: + RuntimeError: If this Session has no working directory, which means + construction did not complete. """ - assert self._working_dir is not None + # A public property, so an explicit raise rather than `assert`: under + # `python -O` the assert vanished and the caller got + # `AttributeError: 'NoneType' object has no attribute 'path'` from inside + # a library property (R5-6). + if self._working_dir is None: + raise RuntimeError( + "This Session has no working directory; its construction did not complete." + ) return self._working_dir.path @property @@ -598,10 +668,44 @@ def cancel_action( """ if self.state != SessionState.RUNNING: raise RuntimeError("No actions are running") - # For the type checker - assert self._runner is not None + # Review22-F3 fix: Snapshot _runner before using it. The state check + # above and the _runner access below are not atomic; a completion + # racing this call could set _runner = None after we pass the state + # check. Snapshotting here avoids a bare AssertionError. + runner = self._runner + if runner is None: + # Race: action completed between state check and here. No-op rather + # than raise, since the caller's intent (cancel the running action) + # is already satisfied. + return - self._runner.cancel(time_limit=time_limit, mark_action_failed=mark_action_failed) + runner.cancel(time_limit=time_limit, mark_action_failed=mark_action_failed) + + def _make_env_script_runner( + self, + *, + environment_script: Optional[EnvironmentScriptModel], + os_env_vars: dict[str, Optional[str]], + symtab: SymbolTable, + preallocated_file_records: Optional[list[_FileRecord]] = None, + ) -> EnvironmentScriptRunner: + """Construct an :class:`EnvironmentScriptRunner` with this Session's + standard wiring (logger, user, working/files directories, and action + callback). Only the script, subprocess environment, symbol table, and + (for wrap hooks) pre-allocated embedded-file records vary between the + call sites.""" + return EnvironmentScriptRunner( + logger=self._logger, + user=self._user, + os_env_vars=os_env_vars, + session_working_directory=self.working_directory, + startup_directory=self.working_directory, + callback=self._action_callback, + environment_script=environment_script, + symtab=symtab, + session_files_directory=self.files_directory, + preallocated_file_records=preallocated_file_records, + ) def enter_environment( self, @@ -609,6 +713,8 @@ def enter_environment( environment: EnvironmentModel, identifier: Optional[EnvironmentIdentifier] = None, os_env_vars: Optional[dict[str, str]] = None, + extra_let_bindings: Optional[list[str]] = None, + step_name: Optional[str] = None, ) -> EnvironmentIdentifier: """Enters an Open Job Description Environment within this Session. This method is non-blocking; it will exit when the subprocess is either confirmed to have @@ -627,10 +733,30 @@ def enter_environment( by values defined in Environments. Key: Environment variable name Value: Value for the environment variable. + extra_let_bindings (Optional[list[str]]): Additional EXPR ``let`` + bindings (RFC 0005) evaluated into the symbol table before the + environment's variables and actions resolve. A step's + environments are entered with the step-level ``let`` bindings + (``Step.let`` on the instantiated Job) so both can reference + them — the v0 counterpart of the per-step resolved symbol + table that openjd-rs threads into enter_environment. + step_name (Optional[str]): The name of the step whose + stepEnvironments are being entered, if any. Seeds + ``Step.Name`` (RFC 0005 EXPR) into the symbol table before + the extra ``let`` bindings evaluate, so step-level bindings + and the environment's variables and actions can reference + it — openjd-rs threads a per-step resolved symbol table into + environment entry, and this is the v0 counterpart. Returns: EnvironmentIdentifier: An identifier by which the Environment is known by to this Session. Pass this identifier to exit_environment() when exiting this Environment. + + Raises: + RuntimeError: If the Session is not in the READY state; if the given + identifier has already been entered in this Session; or, per + RFC 0008, if this Environment defines wrap hooks and another + entered Environment already does. """ if self.state != SessionState.READY: raise RuntimeError("Session must be in the READY state to enter an Environment.") @@ -638,6 +764,21 @@ def enter_environment( raise RuntimeError( f"Environment {identifier} has already been entered in this Session." ) + + # RFC 0008: at most one Environment in the session stack may + # define any wrap hook. Reject the new environment up front if it + # would create a second wrap layer. + if self._environment_defines_any_wrap_hook(environment): + for existing_id in self._environments_entered: + existing = self._environments[existing_id] + if self._environment_defines_any_wrap_hook(existing): + raise RuntimeError( + "RFC 0008: a session may have at most one Environment " + "defining wrap hooks (onWrapEnvEnter / onWrapTaskRun / " + f"onWrapEnvExit). Environment '{existing.name}' already " + f"defines wrap hooks; cannot also enter '{environment.name}'." + ) + log_section_banner(self._logger, f"Entering Environment: {environment.name}") self._reset_action_state() @@ -646,20 +787,82 @@ def enter_environment( identifier = f"{self._session_id}:{uuid.uuid4().hex}" self._environments[identifier] = environment + if extra_let_bindings: + self._environment_extra_let_bindings[identifier] = list(extra_let_bindings) + if step_name is not None: + self._environment_step_names[identifier] = step_name self._environments_entered.append(identifier) self._running_environment_identifier = identifier symtab = self._symbol_table(environment.revision) + # RFC 0005; Template Schemas §7.3.1 (EXPR): the owning step's name. Only EXPR templates + # pass validation referencing Step.Name, so seeding it when known does + # not change non-EXPR behavior. Seeded before the extra `let` bindings + # evaluate so a step-level binding may reference it. + if step_name is not None: + symtab["Step.Name"] = step_name + + # Step-level `let` bindings (RFC 0005) accompany a step's + # environments: evaluate them first so the environment's variables + # and actions can reference them. + if extra_let_bindings: + try: + apply_let_bindings(symtab=symtab, let_bindings=extra_let_bindings) + except ValueError as e: + # ExpressionError and FormatStringError subclass ValueError: + # a binding failed to evaluate (e.g. it referenced an + # undefined symbol). Fail the action through the normal + # failure path rather than raising out of the public API — + # the environment stays in the entered list, exactly as when + # a failing onEnter subprocess leaves it, so the caller's + # cleanup exits it as usual. + self._created_env_vars[identifier] = SimplifiedEnvironmentVariableChanges( + dict[str, str]() + ) + self._fail_action_before_start( + f"Failed to evaluate the extra `let` bindings for {environment.name}: {e}" + ) + return identifier + + # Note: the environment script's own EXPR `let` bindings (RFC 0005) + # are evaluated by the script runner, after embedded-file paths are + # allocated (so bindings can reference Env.File.*). The environment's + # `variables` resolve against the session symbol table without them, + # matching the openjd-rs runtime. + if environment.variables is not None: # We must process the current environment's variables # before we call _evaluate_current_session_env_vars() # otherwise, we will end up running onEnter without # the environment variables of the current environment # being set. - resolved_variables = self._resolve_env_variable_format_strings( - symtab, environment.variables - ) + try: + resolved_variables = self._resolve_env_variable_format_strings( + symtab, environment.variables + ) + except ValueError as e: + # ExpressionError and FormatStringError subclass ValueError. A + # variable's expression failed to evaluate at run time — which + # EXPR (RFC 0005) makes reachable for a template that passed + # validation, e.g. a host function applied to a value only known + # at run time. Fail the action through the normal failure path + # rather than raising out of the public API: the environment is + # already in the entered list, so the caller must be given a + # terminal ActionStatus *and* the identifier in order to exit it. + # + # Seeding the empty change record is required, not tidiness: + # _action_log_filter_callback indexes _created_env_vars by the + # running environment's identifier without a membership test, so + # an openjd_env emitted by this environment's onExit would + # otherwise raise KeyError on the log-forwarding thread. + self._created_env_vars[identifier] = SimplifiedEnvironmentVariableChanges( + dict[str, str]() + ) + self._fail_action_before_start( + f"Failed to resolve the environment variables for {environment.name}: {e}" + ) + return identifier for name, value in resolved_variables.items(): self._logger.info( "Setting: %s=%s", @@ -682,26 +885,89 @@ def enter_environment( self._materialize_path_mapping(environment.revision, action_env_vars, symtab) - # Sets the subprocess running. - # Returns immediately after it has started, or is running - self._action_state = ActionState.RUNNING - self._state = SessionState.RUNNING - # Note: This may fail immediately (e.g. if we cannot write embedded files to disk), - # so it's important to set the action_state to RUNNING before calling enter(), rather - # than after -- enter() itself may end up setting the action state to FAILED. - - self._runner = EnvironmentScriptRunner( - logger=self._logger, - user=self._user, - os_env_vars=action_env_vars, - session_working_directory=self.working_directory, - startup_directory=self.working_directory, - callback=self._action_callback, - environment_script=environment.script, - symtab=symtab, - session_files_directory=self.files_directory, + # Note: RUNNING is set below, immediately before the runner is asked to + # start, and never before `self._runner` exists. `cancel_action()` is a + # cross-thread API guarded only by `state == RUNNING`, so a window where + # the session claims RUNNING with no runner is a window in which a + # cancel is lost. That matters here because the RFC 0008 branch does + # real work first (materializing the inner entity's embedded files, + # evaluating its `let` bindings, allocating the hook's file records). + # Every failure before the launch goes through + # `_fail_action_before_start`, which sets FAILED/READY_ENDING without + # needing a prior RUNNING. + + # RFC 0008: an outer environment's onWrapEnvEnter intercepts an + # inner environment's onEnter. The outer environment's *own* + # onEnter is never wrapped — that's why the lookup excludes the + # environment we just appended (which is at the top of the stack). + on_enter_action = ( + environment.script.actions.onEnter if environment.script is not None else None + ) + wrap_env = ( + self._find_wrap_environment(hook="onWrapEnvEnter") + if on_enter_action is not None + else None ) - self._runner.enter() + # The wrapping environment is whichever earlier-entered env defines + # onWrapEnvEnter — never the env we just entered. + if wrap_env is environment: + wrap_env = None + + if wrap_env is not None: + # The wrapped onEnter resolves against the INNER environment's + # own scope (`symtab`, which carries the step context and + # extra_let_bindings that environment was entered with); the hook + # resolves against its own table, which carries none of them -- see + # _build_wrap_hook_scope. Its own script's lets/files are evaluated + # into that table by the script runner from wrap_env.script. On + # failure the environment stays in the entered list, exactly as + # when enter() itself fails, so the caller's cleanup exits it + # as usual. + hook_symtab = self._build_wrap_hook_scope(environment.revision, symtab) + if not self._try_inject_wrapped_symbols( + scope=EmbeddedFilesScope.ENV, + inner_script=environment.script, + symtab=symtab, + inject=lambda inner_symtab: self._inject_wrapped_env_symbols( + hook_symtab, environment, on_enter_action, inner_symtab=inner_symtab + ), + fail_message=( + f"Failed to resolve the wrapped onEnter action of " + f"{environment.name} for {wrap_env.name}'s onWrapEnvEnter" + ), + ): + return identifier + if not self._seed_wrap_env_scope(hook_symtab, wrap_env): + return identifier + try: + wrap_file_records = self._get_wrap_env_file_records(wrap_env) + except RuntimeError as e: + self._fail_action_before_start( + f"Failed to allocate embedded files for {wrap_env.name}: {e}" + ) + return identifier + self._runner = self._make_env_script_runner( + environment_script=wrap_env.script, + os_env_vars=action_env_vars, + symtab=hook_symtab, + preallocated_file_records=wrap_file_records, + ) + # Sets the subprocess running; returns once it has started, or has + # failed to start. Set RUNNING first: wrap_env_enter() may fail + # immediately (e.g. an embedded file that cannot be written) and + # set the action state to FAILED itself. + self._action_state = ActionState.RUNNING + self._state = SessionState.RUNNING + self._runner.wrap_env_enter() + else: + self._runner = self._make_env_script_runner( + environment_script=environment.script, + os_env_vars=action_env_vars, + symtab=symtab, + ) + self._action_state = ActionState.RUNNING + self._state = SessionState.RUNNING + self._runner.enter() return identifier @@ -733,7 +999,9 @@ def exit_environment( the environments of a step and then run tasks from a different step. Raises: - ValueError - If the given identifier is not that of the next one that must be exited. + RuntimeError: If the Session is not in the READY or READY_ENDING state; + if the given identifier is not known to this Session; or if it is + not the next Environment that must be exited. """ if self.state != SessionState.READY and self.state != SessionState.READY_ENDING: raise RuntimeError( @@ -758,34 +1026,122 @@ def exit_environment( # Must be run _before_ we pop _environments_entered action_env_vars = self._evaluate_current_session_env_vars(os_env_vars) + # RFC 0008: capture the openjd_env list for WrappedAction.Environment + # _before_ the exiting environment is removed from tracking, so the + # list includes that environment's own openjd_env variables — the + # real subprocess environment (action_env_vars, computed above) + # includes them, and the wrapped onExit runs with them in the + # unwrapped case too. + wrapped_session_env_list = self._collect_session_env_list() + # Remove the environment from our tracking since we're now exiting it. del self._environments[identifier] self._environments_entered.pop() + # RFC 0008: drop any embedded-file records reused across this (wrap) + # environment's hook invocations; the files themselves live in the + # session directory and are cleaned up with it. + self._wrap_env_file_records.pop(identifier, None) self._running_environment_identifier = identifier symtab = self._symbol_table(environment.revision) self._materialize_path_mapping(environment.revision, action_env_vars, symtab) - # Sets the subprocess running. - # Returns immediately after it has started, or is running - self._action_state = ActionState.RUNNING - self._state = SessionState.RUNNING - # Note: This may fail immediately (e.g. if we cannot write embedded files to disk), - # so it's important to set the action_state to RUNNING before calling exit(), rather - # than after -- exit() itself may end up setting the action state to FAILED. - self._runner = EnvironmentScriptRunner( - logger=self._logger, - user=self._user, - os_env_vars=action_env_vars, - session_working_directory=self.working_directory, - startup_directory=self.working_directory, - callback=self._action_callback, - environment_script=environment.script, - symtab=symtab, - session_files_directory=self.files_directory, + # Re-seed the owning step's name (Step.Name, RFC 0005 EXPR) and + # re-apply the extra `let` bindings this environment was entered with + # (e.g. the owning step's step-level bindings, RFC 0005) so its onExit + # resolves in the same scope as its onEnter. + exit_step_name = self._environment_step_names.pop(identifier, None) + if exit_step_name is not None: + symtab["Step.Name"] = exit_step_name + exit_extra_let_bindings = self._environment_extra_let_bindings.pop(identifier, None) + if exit_extra_let_bindings: + try: + apply_let_bindings(symtab=symtab, let_bindings=exit_extra_let_bindings) + except ValueError as e: + # ExpressionError and FormatStringError subclass ValueError: + # a binding failed to evaluate. Fail the action through the + # normal failure path rather than raising out of the public + # API — the environment was already removed from tracking + # above, matching how a failing onExit subprocess leaves it. + self._fail_action_before_start( + f"Failed to evaluate the extra `let` bindings for {environment.name}: {e}" + ) + return + + # Note: the environment script's own EXPR `let` bindings (RFC 0005) + # are evaluated by the script runner (after embedded-file path + # allocation); the wrap-interception branch below evaluates them + # itself before resolving the wrapped onExit. + + # Note: RUNNING is set below, immediately before the runner is asked to + # start, and never before `self._runner` exists -- see the note in + # enter_environment for why that window matters to `cancel_action()`. + + # RFC 0008: an outer environment's onWrapEnvExit intercepts an + # inner environment's onExit. The inner environment was already + # popped from ``_environments_entered`` above, so a stack search + # now only turns up genuinely-outer environments. + on_exit_action = ( + environment.script.actions.onExit if environment.script is not None else None + ) + wrap_env = ( + self._find_wrap_environment(hook="onWrapEnvExit") + if on_exit_action is not None + else None ) - self._runner.exit() + + if wrap_env is not None: + # See the onWrapEnvEnter path (_try_inject_wrapped_symbols). On + # failure the environment was already removed from tracking + # above, matching how a failed exit() behaves. + hook_symtab = self._build_wrap_hook_scope(environment.revision, symtab) + if not self._try_inject_wrapped_symbols( + scope=EmbeddedFilesScope.ENV, + inner_script=environment.script, + symtab=symtab, + inject=lambda inner_symtab: self._inject_wrapped_env_symbols( + hook_symtab, + environment, + on_exit_action, + session_env_list=wrapped_session_env_list, + inner_symtab=inner_symtab, + ), + fail_message=( + f"Failed to resolve the wrapped onExit action of " + f"{environment.name} for {wrap_env.name}'s onWrapEnvExit" + ), + ): + return + if not self._seed_wrap_env_scope(hook_symtab, wrap_env): + return + try: + wrap_file_records = self._get_wrap_env_file_records(wrap_env) + except RuntimeError as e: + self._fail_action_before_start( + f"Failed to allocate embedded files for {wrap_env.name}: {e}" + ) + return + self._runner = self._make_env_script_runner( + environment_script=wrap_env.script, + os_env_vars=action_env_vars, + symtab=hook_symtab, + preallocated_file_records=wrap_file_records, + ) + # Set RUNNING first: wrap_env_exit() may fail immediately and set + # the action state to FAILED itself. + self._action_state = ActionState.RUNNING + self._state = SessionState.RUNNING + self._runner.wrap_env_exit() + else: + self._runner = self._make_env_script_runner( + environment_script=environment.script, + os_env_vars=action_env_vars, + symtab=symtab, + ) + self._action_state = ActionState.RUNNING + self._state = SessionState.RUNNING + self._runner.exit() def run_task( self, @@ -794,6 +1150,7 @@ def run_task( task_parameter_values: TaskParameterSet, os_env_vars: Optional[dict[str, str]] = None, log_task_banner: bool = True, + step_name: Optional[str] = None, ) -> None: """Run a Task within the Session. This method is non-blocking; it will exit when the subprocess is either confirmed to have @@ -812,10 +1169,37 @@ def run_task( Value: Value for the environment variable. log_task_banner (bool): Whether to log a banner before running the Task. Default: True + step_name (Optional[str]): The name of the step whose task is being run. + Used by RFC 0008 to populate ``WrappedStep.Name`` in wrap hooks. + Required when a wrap Environment is active. + + Raises: + RuntimeError: If the Session is not in the READY state. + ValueError: If a wrap Environment (RFC 0008) is active and no + ``step_name`` was given. """ if self.state != SessionState.READY: raise RuntimeError("Session must be in the READY state to run a task.") + # Look up the active wrap environment (RFC 0008) up front, before any + # state is reset or anything is logged, so that rejecting the call leaves + # the session exactly as it was — including the previous action's status. + wrap_env = self._find_wrap_environment(hook="onWrapTaskRun") + if wrap_env is not None and step_name is None: + # RFC 0008 defines WrappedStep.Name as the name of the wrapped step, + # and has a minimum length of one — there is no "unknown + # step" value to render. Rendering the empty string instead would + # silently hand a wrap script an empty container or label name, so + # this is reported as what it is: caller misuse, in the same shape + # run_task already reports caller misuse. Raising (rather than + # failing the action) keeps the session usable, so the caller can + # retry with a step name. + raise ValueError( + f"run_task() requires step_name when a wrap environment " + f"('{wrap_env.name}') is active: RFC 0008's WrappedStep.Name " + f"has no value to render without it." + ) + if log_task_banner: log_section_banner(self._logger, "Running Task") @@ -832,27 +1216,104 @@ def run_task( self._reset_action_state() symtab = self._symbol_table(step_script.revision, task_parameter_values) + # RFC 0005; Template Schemas §7.3.1 (EXPR): the running step's name. Only EXPR templates + # pass validation referencing Step.Name, so seeding it when known does + # not change non-EXPR behavior. + if step_name is not None: + symtab["Step.Name"] = step_name action_env_vars = self._evaluate_current_session_env_vars(os_env_vars) self._materialize_path_mapping(step_script.revision, action_env_vars, symtab) - self._runner = StepScriptRunner( - logger=self._logger, - user=self._user, - os_env_vars=action_env_vars, - session_working_directory=self.working_directory, - startup_directory=self.working_directory, - callback=self._action_callback, - script=step_script, - symtab=symtab, - session_files_directory=self.files_directory, - ) - # Sets the subprocess running. - # Returns immediately after it has started, or is running - self._action_state = ActionState.RUNNING - self._state = SessionState.RUNNING - # Note: This may fail immediately (e.g. if we cannot write embedded files to disk), - # so it's important to set the action_state to RUNNING before calling run(), rather - # than after -- run() itself may end up setting the action state to FAILED. - self._runner.run() + + # Note: the step script's EXPR `let` bindings (RFC 0005) are evaluated + # by the script runner, after embedded-file paths are allocated (so + # bindings can reference Task.File.*). The wrap-interception branch + # below evaluates them itself before resolving the wrapped onRun. + + # If a wrap environment is active, inject WrappedAction.* into the symbol + # table and run its hook instead of the step script's onRun (RFC 0008). + if wrap_env is not None: + # RFC 0008 requires WrappedStep.Name, and a wrapped run without a step + # name is already rejected at the top of this method, under this same + # `wrap_env is not None` condition. So this is narrowing for the type + # checker, not a runtime invariant, and a second runtime check here + # would be unreachable -- CodeQL says so, and it is right. Bound once, + # where the requirement has been proven, rather than re-checked. + wrapped_step_name = cast(str, step_name) + # Two separate scopes. The wrapped onRun resolves against the STEP's + # own scope (`symtab`, which carries this task's parameters and the + # running step's name); the hook resolves against its own table, + # which deliberately carries none of that -- see + # _build_wrap_hook_scope. The wrap environment's own lets/files are + # evaluated into the hook's table by the script runner from + # wrap_env.script. + hook_symtab = self._build_wrap_hook_scope(step_script.revision, symtab) + if not self._try_inject_wrapped_symbols( + scope=EmbeddedFilesScope.STEP, + inner_script=step_script, + symtab=symtab, + inject=lambda inner_symtab: self._inject_wrapped_task_symbols( + hook_symtab, step_script, wrapped_step_name, inner_symtab=inner_symtab + ), + fail_message=( + f"Failed to resolve the wrapped Task action for {wrap_env.name}'s " + "onWrapTaskRun" + ), + ): + return + + # Give the hook the step context its OWN environment was entered + # with. Ordering is no longer load-bearing here: `hook_symtab` is a + # separate table and is never the base of + # `_build_wrapped_inner_scope`, so seeding it cannot reach the + # wrapped action's scope whenever it happens. It stays after + # injection to keep all three hook paths reading the same way. + if not self._seed_wrap_env_scope(hook_symtab, wrap_env): + return + + try: + wrap_file_records = self._get_wrap_env_file_records(wrap_env) + except RuntimeError as e: + self._fail_action_before_start( + f"Failed to allocate embedded files for {wrap_env.name}: {e}" + ) + return + self._runner = self._make_env_script_runner( + environment_script=wrap_env.script, + os_env_vars=action_env_vars, + symtab=hook_symtab, + preallocated_file_records=wrap_file_records, + ) + # Note: unlike enter_environment()/exit_environment(), which set + # RUNNING before their first failable step, this path sets it only + # after wrapped-symbol injection has succeeded — so an injection + # failure reports FAILED/READY_ENDING without the session ever + # having been observably RUNNING. Harmless for the documented + # poll-then-check-action_status pattern, but the asymmetry is + # deliberate: there is no runner to own the failure until here. + self._action_state = ActionState.RUNNING + self._state = SessionState.RUNNING + self._runner.wrap_task_run() + else: + # Original path: run the step script directly. + self._runner = StepScriptRunner( + logger=self._logger, + user=self._user, + os_env_vars=action_env_vars, + session_working_directory=self.working_directory, + startup_directory=self.working_directory, + callback=self._action_callback, + script=step_script, + symtab=symtab, + session_files_directory=self.files_directory, + ) + # Sets the subprocess running. + # Returns immediately after it has started, or is running + self._action_state = ActionState.RUNNING + self._state = SessionState.RUNNING + # Note: This may fail immediately (e.g. if we cannot write embedded files to disk), + # so it's important to set the action_state to RUNNING before calling run(), rather + # than after -- run() itself may end up setting the action state to FAILED. + self._runner.run() def _run_task_without_session_env( self, @@ -864,6 +1325,11 @@ def _run_task_without_session_env( ) -> None: """Private API to run a task within the session. This method directly use os_env_vars passed in without applying additional session env setup. + + Note: unlike :meth:`run_task`, this deliberately does *not* dispatch RFC + 0008 wrap hooks, and does not seed ``Step.Name``. A caller that uses this + while a wrap Environment is entered runs the task unwrapped. Callers must + move to :meth:`run_task` to participate in wrap actions. """ if self.state != SessionState.READY: raise RuntimeError("Session must be in the READY state to run a task.") @@ -891,6 +1357,10 @@ def _run_task_without_session_env( action_env_vars.update(**os_env_vars) self._materialize_path_mapping(step_script.revision, action_env_vars, symtab) + + # Note: the step script's EXPR `let` bindings (RFC 0005) are evaluated + # by the script runner, after embedded-file paths are allocated. + self._runner = StepScriptRunner( logger=self._logger, user=self._user, @@ -1059,39 +1529,519 @@ def _symbol_table( ) -> SymbolTable: """Construct a SymbolTable, with fully qualified value names, suitable for running a Script.""" - def processed_parameter_value(param: ParameterValue) -> str: - if param.type == ParameterValueType.PATH and self._path_mapping_rules is not None: + def apply_mapping(path: str) -> str: + if self._path_mapping_rules is not None: # Apply path mapping rules in the order given until one does a replacement for rule in self._path_mapping_rules: - changed, result = rule.apply(path=param.value) + changed, result = rule.apply(path=path) if changed: return result + return path + + def processed_parameter_value(param: ParameterValue) -> Any: + if param.type == ParameterValueType.PATH: + return apply_mapping(param.value) + if param.type == ParameterValueType.LIST_PATH and isinstance(param.value, list): + # openjd-rs maps each element of a LIST[PATH] parameter at + # session scope; mirror that element-wise mapping. + return [apply_mapping(p) for p in param.value] return param.value + def record_expr_types( + expr_types: dict[str, str], key: str, raw_key: str, ptype: ParameterValueType + ) -> None: + """Record the EXPR types for a parameter's ``Param.*``/``RawParam.*`` + (or ``Task.Param.*``/``Task.RawParam.*``) symbols, mirroring the + openjd-rs session-scope symbol table: + + - ``Param.*`` carries the declared type (a PATH is a host-format + path value with path mapping applied). + - ``RawParam.*`` carries the raw string form: PATH stays a plain + string and LIST[PATH] is a LIST[STRING]. + - CHUNK[INT] range strings stay strings (engine inference). + + The types only affect EXPR-extension expression evaluation; the + legacy (non-EXPR) interpolation path ignores them. + """ + if ptype == ParameterValueType.CHUNK_INT: + return + expr_types[key] = ptype.value + if ptype == ParameterValueType.PATH: + pass # raw form stays an (inferred) plain string + elif ptype == ParameterValueType.LIST_PATH: + expr_types[raw_key] = ParameterValueType.LIST_STRING.value + else: + expr_types[raw_key] = ptype.value + if version == SpecificationRevision.v2023_09: symtab = SymbolTable() - symtab[ValueReferenceConstants_2023_09.WORKING_DIRECTORY.value] = str( - self.working_directory - ) + # The session is host scope: enable EXPR host-context functions + # (e.g. apply_path_mapping) with this session's rules. + symtab.expr_host_rules = self._expr_host_rules + working_dir_key = ValueReferenceConstants_2023_09.WORKING_DIRECTORY.value + symtab[working_dir_key] = str(self.working_directory) + # Session.WorkingDirectory is a host-format path value in openjd-rs. + symtab.expr_types[working_dir_key] = ParameterValueType.PATH.value + # RFC 0005; Template Schemas §7.3.1 (EXPR): the job's resolved name. Only templates + # declaring EXPR pass validation referencing Job.Name, so seeding + # it whenever known does not change non-EXPR behavior. + if self._job_name is not None: + symtab["Job.Name"] = self._job_name for param_name, param_props in self._job_parameter_values.items(): - symtab[ + raw_key = ( f"{ValueReferenceConstants_2023_09.JOB_PARAMETER_RAWPREFIX.value}.{param_name}" - ] = param_props.value - symtab[ - f"{ValueReferenceConstants_2023_09.JOB_PARAMETER_PREFIX.value}.{param_name}" - ] = processed_parameter_value(param_props) + ) + key = f"{ValueReferenceConstants_2023_09.JOB_PARAMETER_PREFIX.value}.{param_name}" + symtab[raw_key] = param_props.value + symtab[key] = processed_parameter_value(param_props) + record_expr_types(symtab.expr_types, key, raw_key, param_props.type) if task_parameter_values: for param_name, param_props in task_parameter_values.items(): - symtab[ - f"{ValueReferenceConstants_2023_09.TASK_PARAMETER_RAWPREFIX.value}.{param_name}" - ] = param_props.value - symtab[ - f"{ValueReferenceConstants_2023_09.TASK_PARAMETER_PREFIX.value}.{param_name}" - ] = processed_parameter_value(param_props) + raw_key = f"{ValueReferenceConstants_2023_09.TASK_PARAMETER_RAWPREFIX.value}.{param_name}" + key = f"{ValueReferenceConstants_2023_09.TASK_PARAMETER_PREFIX.value}.{param_name}" + symtab[raw_key] = param_props.value + symtab[key] = processed_parameter_value(param_props) + record_expr_types(symtab.expr_types, key, raw_key, param_props.type) return symtab else: raise NotImplementedError(f"Schema version {str(version.value)} is not supported.") + def _build_expr_host_rules(self) -> Optional[list[Any]]: + """Convert this session's path mapping rules to their engine + (``openjd.expr.PathMappingRule``) form for EXPR host-context + evaluation. Returns an empty list when the session has no rules (the + session is still host scope), or ``None`` when the engine bindings + are unavailable (pre-EXPR openjd-model).""" + try: + from openjd.expr import PathFormat as ExprPathFormat + from openjd.expr import PathMappingRule as ExprPathMappingRule + except ImportError: + return None + rules = [] + for rule in self._path_mapping_rules or []: + rules.append( + ExprPathMappingRule( + source_path_format=getattr(ExprPathFormat, rule.source_path_format.name), + source_path=str(rule.source_path), + destination_path=str(rule.destination_path), + ) + ) + return rules + + # ------------------------------------------------------------------ + # RFC 0008 wrap-action helpers + # ------------------------------------------------------------------ + + def _find_wrap_environment(self, *, hook: str) -> Optional[EnvironmentModel]: + """Walk the environment stack (innermost first) and return the + active wrapping environment for ``hook`` (one of ``onWrapEnvEnter``, + ``onWrapTaskRun``, or ``onWrapEnvExit``). + + Per RFC 0008 the session is only valid with at most one + wrap-defining environment in the stack, so the first match is + always the one that applies.""" + if hook not in WRAP_HOOK_ACTION_NAMES: + raise ValueError(f"Unknown wrap hook name: {hook}") + for env_id in reversed(self._environments_entered): + env = self._environments[env_id] + if ( + env.script is not None + and hasattr(env.script.actions, hook) + and getattr(env.script.actions, hook) is not None + ): + return env + return None + + def _wrap_env_identifier(self, wrap_env: EnvironmentModel) -> EnvironmentIdentifier: + """The entered-stack identifier of ``wrap_env``. + + Raises: + RuntimeError: if the environment is not in the entered stack. + """ + identifier = next( + ( + env_id + for env_id in self._environments_entered + if self._environments[env_id] is wrap_env + ), + None, + ) + if identifier is None: # pragma: no cover - guarded by _find_wrap_environment + raise RuntimeError( + f"Wrap environment '{wrap_env.name}' is not in this Session's entered stack." + ) + return identifier + + def _build_wrap_hook_scope( + self, version: SpecificationRevision, session_symtab: SymbolTable + ) -> SymbolTable: + """The scope an RFC 0008 wrap hook resolves in. + + Built fresh from session scope rather than derived from the inner + entity's table, and that distinction is the whole point. The inner + entity's table carries symbols belonging to the wrapped work: a task's + ``Task.Param.*``/``Task.RawParam.*``, the running step's ``Step.Name``, + and the ``extra_let_bindings`` the *inner* environment was entered with. + A wrap environment must not be able to read any of them. + + :meth:`_build_wrapped_inner_scope` already blocks the wrap -> inner + direction by resolving the wrapped action against a copy. This is the + other direction, which was open: the hook used to resolve against the + inner entity's own table, so ``{{Task.Param.Frame}}`` in a hook's args + resolved to the wrapped task's frame number and ``{{Step.Name}}`` to the + running step. RFC 0008 supplies ``WrappedStep.Name`` precisely because + ``Step.Name`` is not meant to be reachable from a hook, and the model + does not reject either reference in an environment script, so this was + the only gate. + + What a hook legitimately gets: session scope (``Session.*``, + ``Job.Name``, ``Param.*``/``RawParam.*``), the path-mapping symbols, the + wrap environment's *own* enter-time step context (applied afterwards by + :meth:`_seed_wrap_env_scope`), the ``WrappedAction.*`` overlay, and its + own script-level ``let`` bindings and embedded files (evaluated by the + runner). + + The path-mapping symbols are copied rather than re-materialized: the + rules file has already been written for this action, and both scopes must + name the same file. + """ + hook_symtab = self._symbol_table(version) + for key in ( + ValueReferenceConstants_2023_09.HAS_PATH_MAPPING_RULES.value, + ValueReferenceConstants_2023_09.PATH_MAPPING_RULES_FILE.value, + ): + # _materialize_path_mapping sets the value and its EXPR type + # together, so one membership check covers both. The check itself is + # not decoration: this method is also called on a table that has not + # been through path mapping. + if key in session_symtab: + hook_symtab[key] = session_symtab[key] + hook_symtab.expr_types[key] = session_symtab.expr_types[key] + return hook_symtab + + def _seed_wrap_env_scope(self, symtab: SymbolTable, wrap_env: EnvironmentModel) -> bool: + """Re-seed the scope a wrap hook resolves in with the step context the + wrap environment was *entered* with (RFC 0005 step-level ``let`` + bindings and ``Step.Name``). + + A wrap hook resolves in the wrap environment's own scope, and in + openjd-rs that scope is the environment's frozen enter-time resolved + symbol table — so a step environment that defines wrap hooks carries + the owning step's step-level ``let`` bindings into every hook + invocation. Python builds a fresh session-scope table per action, so + those bindings have to be re-applied here from what + :meth:`enter_environment` remembered (it already keeps them to + re-apply on the exit side). + + Call this *after* the wrapped action's own scope has been built, so + the wrap environment's bindings cannot reach the wrapped action's + resolution — only the hook's. + + Returns: + True on success. On failure the action has already been failed + through :meth:`_fail_action_before_start` and the caller must + return. + """ + identifier = self._wrap_env_identifier(wrap_env) + step_name = self._environment_step_names.get(identifier) + if step_name is not None: + symtab["Step.Name"] = step_name + let_bindings = self._environment_extra_let_bindings.get(identifier) + if not let_bindings: + return True + try: + apply_let_bindings(symtab=symtab, let_bindings=let_bindings) + except ValueError as e: + # ExpressionError and FormatStringError subclass ValueError. These + # bindings already evaluated successfully when the environment was + # entered, so this is unlikely — but it must not raise out of the + # public API. + self._fail_action_before_start( + f"Failed to evaluate the extra `let` bindings for {wrap_env.name}: {e}" + ) + return False + return True + + def _get_wrap_env_file_records(self, wrap_env: EnvironmentModel) -> Optional[list[_FileRecord]]: + """Return the wrap environment's embedded-file records, allocating + their on-disk paths on first use and reusing them for every + subsequent wrap-hook invocation (see ``_wrap_env_file_records``). + Returns ``None`` when the wrap environment has no embedded files. + + The allocation only reserves the paths (defining the symbols into a + throwaway table); the per-invocation symbol definitions and content + writes happen in the runner against that invocation's symbol table. + + Raises: + RuntimeError: if a file path could not be allocated. + """ + if wrap_env.script is None or wrap_env.script.embeddedFiles is None: + return None + # The wrap env's identifier: it is in the entered stack (that's how + # _find_wrap_environment found it). + identifier = self._wrap_env_identifier(wrap_env) + records = self._wrap_env_file_records.get(identifier) + if records is None: + file_writer = EmbeddedFiles( + logger=self._logger, + scope=EmbeddedFilesScope.ENV, + session_files_directory=self.files_directory, + user=self._user, + ) + # Paths only: symbols are defined (and logged) per wrap-hook + # invocation via register_file_paths, against that invocation's + # own symbol table. + records = file_writer.allocate_records(wrap_env.script.embeddedFiles) + self._wrap_env_file_records[identifier] = records + return records + + def _environment_defines_any_wrap_hook(self, env: EnvironmentModel) -> bool: + """``True`` iff ``env``'s script declares any of the three wrap + hooks. Used by the single-wrap-layer validation in + :meth:`enter_environment`.""" + if env.script is None: + return False + return any( + hasattr(env.script.actions, name) and getattr(env.script.actions, name) is not None + for name in WRAP_HOOK_ACTION_NAMES + ) + + def _build_wrapped_inner_scope( + self, + scope: EmbeddedFilesScope, + let_bindings: Optional[list[str]], + embedded_files: Optional[Any], + base: SymbolTable, + ) -> SymbolTable: + """Build the scope a wrapped action would have resolved against had + it run unwrapped: a copy of ``base`` (the session-scope table) plus + the inner entity's embedded files and script-level ``let`` bindings, + in the same order the runners use (allocate file paths → evaluate + lets → write file contents, so a file's ``data`` can reference + let-bound values and a binding can reference ``*.File.*``). + + ``WrappedAction.*`` values are resolved against this table so that + names defined by the WRAPPING environment (its ``let`` bindings) can + never leak into the wrapped action's resolved command/args — and, + symmetrically, the inner entity's lets never apply to the hook's own + resolution scope. Mirrors openjd-rs's ``build_wrapped_inner_scope``. + + Raises: + ValueError (FormatStringError/ExpressionError): a binding or file + reference did not resolve. + RuntimeError: an embedded file could not be written to disk. + """ + symtab = SymbolTable(source=base) + if embedded_files: + file_writer = EmbeddedFiles( + logger=self._logger, + scope=scope, + session_files_directory=self.files_directory, + user=self._user, + ) + records = file_writer.allocate_file_paths(embedded_files, symtab) + if let_bindings: + apply_let_bindings(symtab=symtab, let_bindings=let_bindings) + file_writer.write_file_contents(records, symtab) + elif let_bindings: + apply_let_bindings(symtab=symtab, let_bindings=let_bindings) + return symtab + + def _try_inject_wrapped_symbols( + self, + *, + scope: EmbeddedFilesScope, + inner_script: Any, + symtab: SymbolTable, + inject: Callable[[SymbolTable], None], + fail_message: str, + ) -> bool: + """Common wrap-interception step for the three RFC 0008 hooks: build + the wrapped (inner) entity's own resolution scope — its ``let`` + bindings and embedded files on a COPY of the session table, so the + hook's own scope never sees them (openjd-rs #277) — and call + ``inject`` with it to populate the ``WrappedAction.*`` symbols in + ``symtab``, the hook's scope. + + On failure (e.g. a binding or embedded file did not resolve or + write, or a FEATURE_BUNDLE_1 timeout/notifyPeriod format string did + not resolve to an integer) the action fails through the normal + failure path (:meth:`_fail_action_before_start` with + ``fail_message``) rather than raising out of the public API, and + ``False`` is returned so the caller can bail out.""" + try: + inner_symtab = self._build_wrapped_inner_scope( + scope, + inner_script.let if inner_script is not None else None, + inner_script.embeddedFiles if inner_script is not None else None, + symtab, + ) + inject(inner_symtab) + except (FormatStringError, ValueError, RuntimeError) as e: + self._fail_action_before_start(f"{fail_message}: {e}") + return False + return True + + def _collect_session_env_list(self) -> list[str]: + """Collect all session-defined variables across the active + environment stack as ``["KEY=value", ...]`` for + ``WrappedAction.Environment``. + + Session-defined variables are ``openjd_env`` stdout definitions + *and* entered environments' declarative ``variables:`` maps + (openjd-rs #277); host-inherited variables remain intentionally + excluded per RFC 0008. + + The per-environment changes are flattened cumulatively, in + environment-entry order, so the list reflects the *effective* + state exactly like the real subprocess environment does: a later + set overrides an earlier value for the same name (one entry, not + two), and a later unset removes the name entirely. This mirrors + the Rust runtime's single cumulative ``env_vars`` map.""" + effective: dict[str, Optional[str]] = {} + for env_id in self._environments_entered: + if env_id in self._created_env_vars: + changes = self._created_env_vars[env_id] + # effective_items() is insertion-ordered, holding both the + # environment's declarative `variables:` seed and any + # openjd_env sets/unsets, so the list order is deterministic. + for key, value in changes.effective_items().items(): + effective[key] = value + return [f"{key}={value}" for key, value in effective.items() if value is not None] + + def _resolve_action_timeout(self, action: Any, symtab: SymbolTable) -> Optional[int]: + """Return the wrapped action's timeout as an int (seconds), or + ``None`` if none was specified — ``WrappedAction.Timeout`` is typed + ``int?`` (RFC 0008), following the EXPR semantics for optional + data, so whole-field forwarding + (``timeout: "{{WrappedAction.Timeout}}"``) drops the field when the + wrapped action has no timeout. + + A resolved format-string value must be a positive integer, matching + the openjd-rs runtime and the enforcement path in + :meth:`ScriptRunnerBase._run_action` (a non-positive value raises + ``ValueError``, failing the action through the caller's normal + failure path).""" + return resolve_optional_int_field(action.timeout, symtab, ge=1, description="timeout") + + def _inject_wrapped_cancelation_symbols( + self, symtab: SymbolTable, action: Any, *, is_task_run: bool, inner_symtab: SymbolTable + ) -> None: + """Populate ``WrappedAction.Cancelation.Mode`` and + ``WrappedAction.Cancelation.NotifyPeriodInSeconds`` from the wrapped + action's ```` (RFC 0008 follow-up, + openjd-specifications#148). + + ``Mode`` is typed ``string?``: ``"TERMINATE"``, + ``"NOTIFY_THEN_TERMINATE"``, or ``None`` (rendering as + ``null``/empty in format strings) when the wrapped action defines + no ```` — the null case is deliberately distinct from + an explicit ``TERMINATE`` so wrap scripts can tell "author declared + TERMINATE" apart from "author declared nothing". + + ``NotifyPeriodInSeconds`` is typed ``int?``: the effective grace + period when the mode is ``NOTIFY_THEN_TERMINATE``, applying the + Template Schemas 5.3.2 defaults (120 seconds for a task's ``onRun``, + 30 otherwise) when the wrapped action omits the field — i.e. the + value the runtime would have enforced in the unwrapped case. It is + ``None`` (rendering as ``null``/empty in format strings) when the + mode is ``TERMINATE`` or no ```` is defined, so "not + applicable" is not conflated with a zero-length notify period. + """ + # Resolution goes through the same helper the enforcement path uses + # (resolve_effective_cancelation) — including a wrapped action + # whose own mode is deferred (a format string) — so the value a + # wrap script sees is always the value the runtime would enforce. + # Direct attribute access, not a getattr default: a model rename must fail + # loudly rather than silently tell every wrap script that the wrapped + # action declared no cancelation. See resolve_effective_cancelation. + mode, notify_period = resolve_effective_cancelation(action.cancelation, inner_symtab) + if mode == CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value and notify_period is None: + notify_period = ( + TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS + if is_task_run + else ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS + ) + symtab["WrappedAction.Cancelation.Mode"] = mode + symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] = notify_period + + def _inject_wrapped_env_symbols( + self, + symtab: SymbolTable, + environment: EnvironmentModel, + inner_action: Any, + session_env_list: Optional[list[str]] = None, + *, + inner_symtab: SymbolTable, + ) -> None: + """Populate ``WrappedAction.*`` and ``WrappedEnv.Name`` for + ``onWrapEnvEnter`` / ``onWrapEnvExit`` scripts (RFC 0008). + + The wrapped action's command/args/timeout/cancelation resolve + against ``inner_symtab`` — the scope the action would have used had + it run unwrapped (the inner environment's own lets and embedded + files) — while the results are written into ``symtab``, the hook's + own scope. Keeping the two apart means a wrapper-defined name can + never leak into the wrapped action's resolved values, and vice + versa (openjd-rs #277). + + ``session_env_list`` overrides the collected session env list when + the caller must capture it at a different point in time — the + ``onWrapEnvExit`` path collects it before the exiting environment + is removed from tracking, so the wrapped environment's own + variables are included.""" + command = inner_action.command.resolve(symtab=inner_symtab) + # RFC 0005 §1.3.2 typed argument semantics (null skip, list + # flattening), shared with the enforcement path + # (ScriptRunnerBase._run_action) so the hook sees exactly the + # arguments the wrapped action would have run with unwrapped — + # mirroring openjd-rs's seed_wrapped_action_symbols, which resolves + # via the same resolve_action_args as the runner. + args = resolve_action_arg_values(inner_action.args, inner_symtab) + symtab["WrappedAction.Command"] = command + symtab["WrappedAction.Args"] = args + symtab["WrappedAction.Environment"] = ( + session_env_list if session_env_list is not None else self._collect_session_env_list() + ) + symtab["WrappedAction.Timeout"] = self._resolve_action_timeout(inner_action, inner_symtab) + self._inject_wrapped_cancelation_symbols( + symtab, inner_action, is_task_run=False, inner_symtab=inner_symtab + ) + symtab["WrappedEnv.Name"] = environment.name + + def _inject_wrapped_task_symbols( + self, + symtab: SymbolTable, + step_script: StepScriptModel, + step_name: str, + *, + inner_symtab: SymbolTable, + ) -> None: + """Populate ``WrappedAction.*`` and ``WrappedStep.Name`` for + ``onWrapTaskRun`` (RFC 0008). + + Resolves the step script's ``onRun`` command and args format + strings against ``inner_symtab`` — the step's own scope (its lets + and embedded files), see ``_inject_wrapped_env_symbols`` — producing + concrete values the wrap action can safely shell-quote with + ``repr_sh()``. ``WrappedAction.Args`` is stored as a native + ``list[str]`` so that ``repr_sh(WrappedAction.Args)`` quotes each + element individually.""" + assert isinstance(step_script, StepScript_2023_09) + on_run = step_script.actions.onRun + + symtab["WrappedAction.Command"] = on_run.command.resolve(symtab=inner_symtab) + # RFC 0005 §1.3.2 typed argument semantics (null skip, list + # flattening), shared with the enforcement path — see + # _inject_wrapped_env_symbols. + symtab["WrappedAction.Args"] = resolve_action_arg_values(on_run.args, inner_symtab) + symtab["WrappedAction.Environment"] = self._collect_session_env_list() + symtab["WrappedAction.Timeout"] = self._resolve_action_timeout(on_run, inner_symtab) + self._inject_wrapped_cancelation_symbols( + symtab, on_run, is_task_run=True, inner_symtab=inner_symtab + ) + symtab["WrappedStep.Name"] = step_name + def _openjd_session_root_dir(self) -> Path: """ Returns (and creates if necessary) the top-level directory where Open Job Description step session @@ -1100,18 +2050,22 @@ def _openjd_session_root_dir(self) -> Path: if self._session_root_directory is not None: return self._session_root_directory - tempdir = Path(custom_gettempdir(self._logger)) - - # Note: If this doesn't have group permissions, then we will be unable to access files - # under this directory if the default group of the current user is the group that - # is shared with a job user. The group permissions override the world permissions - # when the accessor is in the group. - # 0o755 - mode = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH - tempdir.mkdir(exist_ok=True, mode=mode) - # tempdir might already exist with the incorrect permissions, so set the permissions. - os.chmod(tempdir, mode=mode) - return tempdir + # custom_gettempdir() owns this directory's lifecycle end to end: it + # creates it, refuses to return a path that is a link or that another user + # owns (_validate_temp_dir_ownership), and guarantees + # OPENJD_TEMPDIR_MODE. The mode matters and is not merely cosmetic -- if + # the root lacked group/other search permission we would be unable to + # reach files under it when this process' default group is the group + # shared with a job user, because group permissions override world + # permissions for a member of the group. + # + # This used to re-apply the mode itself, with the constant duplicated. The + # duplicate is gone rather than merely single-sourced: two creators of one + # shared path is what let an unvalidated one hand out a directory the + # other then chmod'ed. + # + # Raises: RuntimeError + return Path(custom_gettempdir(self._logger)) def _create_working_directory(self) -> TempDir: """Creates and returns the temporary working directory for this Session""" @@ -1173,11 +2127,21 @@ def _materialize_path_mapping( else: rules_dict = dict() symtab[ValueReferenceConstants_2023_09.HAS_PATH_MAPPING_RULES.value] = "false" + # RFC 0005; Template Schemas §7.3: for EXPR evaluation Session.HasPathMappingRules is a + # boolean and Session.PathMappingRulesFile is a path, matching + # openjd-rs's typed session symbols. The legacy (non-EXPR) + # interpolation path ignores these types and keeps the string forms. + symtab.expr_types[ValueReferenceConstants_2023_09.HAS_PATH_MAPPING_RULES.value] = ( + ParameterValueType.BOOL.value + ) rules_json = json.dumps(rules_dict) file_handle, filename = mkstemp(dir=self.working_directory, suffix=".json", text=True) os.close(file_handle) write_file_for_user(Path(filename), rules_json, self._user) symtab[ValueReferenceConstants_2023_09.PATH_MAPPING_RULES_FILE.value] = str(filename) + symtab.expr_types[ValueReferenceConstants_2023_09.PATH_MAPPING_RULES_FILE.value] = ( + ParameterValueType.PATH.value + ) def _resolve_env_variable_format_strings( self, symtab: SymbolTable, variables: "EnvironmentVariableObject" @@ -1216,18 +2180,36 @@ def _action_log_filter_callback( if cancel_action_mark_failed: # Cancel the action and pass the failure message - self.cancel_action(mark_action_failed=True) + self._cancel_running_action_as_failed() elif kind == ActionMessageKind.ENV: if self._running_environment_identifier is None: - # Ignore the message if we're not running an environment + # Ignore the message if we're not running an environment. + # + # Per How-Jobs-Are-Run, `openjd_env` "can only be emitted by the + # Action for entering an Environment" — so a task's onRun cannot + # define a session variable, and neither can an RFC 0008 + # onWrapTaskRun hook, which stands in for one. That keeps + # wrapping transparent: a task that prints an `openjd_env:` line + # behaves the same wrapped and unwrapped. + # + # This is deliberate, not an oversight, and it is a known + # divergence from openjd-rs, which records such a variable in the + # map that feeds WrappedAction.Environment while still not + # applying it to any subprocess environment — advertising a + # variable the wrapped context does not have. The spec does not + # settle the case (it also says a wrap script MAY emit these + # macros directly) and no conformance fixture covers it; filed + # upstream. Logged at debug so an author chasing a silent no-op + # has something to find, without adding noise to every task log. + self._log_discarded_env_macro(kind, value) return if cancel_action_mark_failed: # Assert for the type checker; the type is guaranteed by the ActionMonitoringFilter assert isinstance(value, str) # Cancel the action and pass the failure message - self.cancel_action(mark_action_failed=True) + self._cancel_running_action_as_failed() self._action_fail_message = value return @@ -1241,7 +2223,9 @@ def _action_log_filter_callback( return elif kind == ActionMessageKind.UNSET_ENV: if self._running_environment_identifier is None: - # Ignore the message if we're not running an environment + # Ignore the message if we're not running an environment. + # See the ENV branch above for why this is deliberate. + self._log_discarded_env_macro(kind, value) return if cancel_action_mark_failed: @@ -1249,7 +2233,7 @@ def _action_log_filter_callback( assert isinstance(value, str) # Cancel the action and pass the failure message - self.cancel_action(mark_action_failed=True) + self._cancel_running_action_as_failed() self._action_fail_message = value return @@ -1266,9 +2250,81 @@ def _action_log_filter_callback( if self._callback: action_status = self.action_status - # for the type checker - assert action_status is not None - self._callback(self._session_id, action_status) + # R5-6: a plain check, not `assert`. Every branch above sets + # `_action_state`, so this is non-None in practice -- but it is read + # on the stdout-forwarding thread, where an AssertionError unwinds + # LoggingSubprocess.run() before the child is waited on. Skipping the + # notification is strictly better than losing the process. + if action_status is not None: + self._callback(self._session_id, action_status) + + def _cancel_running_action_as_failed(self) -> None: + """Cancel the running action and report it as failed, if one is running. + + This is reached from the log-forwarding thread whenever a malformed + OpenJD stdout macro is seen. The state guard matters because that filter + is attached to the session logger, so it also sees lines the *session + itself* logs while no action is running — a task parameter whose name + starts with ``openjd_env`` is enough — and ``cancel_action()`` raises + unless an action is in flight. Without the guard that RuntimeError + propagates out of ``logger.info`` and out of the public API. + """ + if self.state != SessionState.RUNNING: + return + self.cancel_action(mark_action_failed=True) + + def _log_discarded_env_macro(self, kind: ActionMessageKind, value: Any) -> None: + """Record, at debug level, that an environment-variable stdout macro was + ignored because the running Action is not an Environment's entry Action. + + Debug rather than a warning on purpose: ``_reset_action_state`` clears + the running-environment identifier for every task, and this callback + cannot tell an RFC 0008 wrap hook from an ordinary task action — so a + warning here would fire for every existing job whose task happens to + print an ``openjd_env:`` line. + """ + name = value.get("name") if isinstance(value, dict) else value + self._logger.debug( + "Ignoring %s for '%s': environment variables can only be defined by " + "the Action that enters an Environment.", + kind.name.lower(), + name, + ) + + def _fail_action_before_start(self, message: str) -> None: + """Mark the pending action as FAILED before any runner/subprocess + exists (RFC 0008: e.g. when resolving the wrapped action's format + strings for ``WrappedAction.*`` injection fails). + + Mirrors the failure branch of :meth:`_action_callback` — the + session transitions to READY_ENDING so the caller can exit the + entered environments — but does not require ``self._runner``. + """ + self._logger.error(message) + self._action_fail_message = message + self._action_exit_code = None + self._action_state = ActionState.FAILED + self._state = SessionState.READY_ENDING + if self._callback: + action_status = self.action_status + # R5-6: a plain check, not `assert` -- `_action_state` is assigned + # immediately above, so None here would be an internal error, and + # skipping the callback beats raising out of a failure path. + if action_status is None: # pragma: no cover - defensive + return + # R4-6 fix: Isolate consumer-callback exceptions. Same pattern as + # _fail_action in ScriptRunnerBase and _on_process_exit. A consumer + # that raises must not turn a handled resolution failure into an + # exception escaping the public Session API. + try: + self._callback(self._session_id, action_status) + except Exception as exc: + self._logger.error( + f"Exception in session callback: {exc}", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO + ), + ) def _action_callback(self, state: ActionState) -> None: """This callback is invoked: @@ -1282,12 +2338,29 @@ def _action_callback(self, state: ActionState) -> None: We can be certain that the process is no longer running when this is called. """ - # For the type checker - assert self._runner is not None + # R5-6: a bound read with an explicit check rather than `assert`. This is + # invoked from the runner's own completion path, so `_runner` is set -- + # but `cleanup()` clears it, and this runs on the pool worker where an + # AssertionError reaches nobody. Under `python -O` the assert vanished and + # the very next line became an AttributeError in the same blind spot. + runner = self._runner + if runner is None: # pragma: no cover - defensive + return - self._action_exit_code = self._runner.exit_code + self._action_exit_code = runner.exit_code self._action_state = state + # F5 fix: Snapshot action_status BEFORE publishing READY. If we set + # _state = READY first, another thread polling session.state could see + # READY but action_status would still reflect the old (stale or + # incomplete) snapshot. By snapshotting here, the callback receives the + # definitive ActionStatus that corresponds to the terminal state. + # + # Note: We snapshot unconditionally (not guarded by `if self._callback`) + # because action_status is cheap and some tests check exact callback + # invocation patterns including the __bool__ check count. + action_status = self.action_status + if state != ActionState.RUNNING: # Decide which between-action state to enter. if self._ending_only or self._action_state != ActionState.SUCCESS: @@ -1297,10 +2370,7 @@ def _action_callback(self, state: ActionState) -> None: else: self._state = SessionState.READY - if self._callback: - action_status = self.action_status - # for the type checker - assert action_status is not None + if self._callback and action_status is not None: self._callback(self._session_id, action_status) def _evaluate_current_session_env_vars( diff --git a/src/openjd/sessions/_subprocess.py b/src/openjd/sessions/_subprocess.py index 9b21a4ef..a48bd02d 100644 --- a/src/openjd/sessions/_subprocess.py +++ b/src/openjd/sessions/_subprocess.py @@ -9,7 +9,7 @@ from datetime import timedelta from pathlib import Path from queue import Queue, Empty -from subprocess import DEVNULL, PIPE, STDOUT, Popen, list2cmdline, run +from subprocess import DEVNULL, PIPE, STDOUT, Popen, TimeoutExpired, list2cmdline, run from threading import Event, Thread from typing import Callable, Literal, Optional, Sequence, cast, Any @@ -49,6 +49,15 @@ LOG_LINE_MAX_LENGTH = 64 * 1000 # Start out with 64 KB, can increase if needed STDOUT_END_GRACETIME_SECONDS = 5 +ABANDONED_PROCESS_REAP_TIMEOUT_SECONDS = 30 +"""How long to wait for a subprocess we are abandoning to actually exit. + +Only used on the path where :meth:`LoggingSubprocess.run` is leaving without +having waited on the child -- see :meth:`LoggingSubprocess._reap`. Bounded rather +than infinite because that path runs on the runner's pool worker: a child that +somehow survives SIGKILL (uninterruptible I/O, say) must not hang the runner's +future as well as leaking the process.""" + class LoggingSubprocess(object): """A process whose stdout/stderr lines are sent to a given Logger.""" @@ -119,8 +128,14 @@ def exit_code(self) -> Optional[int]: # has been set once the subprocess has completed running. Don't poll here... # we only want to make the returncode available after the run method has # completed its work. - if self._process is not None: - return self._process.returncode + # + # Bind once (same reason as notify()/terminate()): this property is read + # cross-thread via Session.action_status while run()'s `finally` clears + # `_process`, so a None check followed by a separate attribute read raised + # AttributeError out of a public property. + proc = self._process + if proc is not None: + return proc.returncode return self._returncode @property @@ -177,12 +192,36 @@ def run(self) -> None: # Would use is_posix(), but it doesn't short-circuit mypy which then complains # about os.getpgid not being a valid attribute. if not sys.platform == "win32": - if not self._user or self._user.is_process_user(): - self._sudo_child_process_group_id = os.getpgid(self._process.pid) - else: - self._sudo_child_process_group_id = find_sudo_child_process_group_id( - logger=self._logger, - sudo_process=self._process, + # A trivial command can exit before we get here. Looking up the + # process group of an already-reaped child raises ProcessLookupError + # (ESRCH), which must not fail the action: the child ran, and its + # exit code is still collected below. + try: + if not self._user or self._user.is_process_user(): + self._sudo_child_process_group_id = os.getpgid(self._process.pid) + else: + self._sudo_child_process_group_id = find_sudo_child_process_group_id( + logger=self._logger, + sudo_process=self._process, + ) + except ProcessLookupError: + # R5-9 fix: leave this as None -- "no process group known" -- + # rather than substituting the pid. The lookup failed *because* + # the process is gone, so its pid is dead: signalling it is at + # best a no-op, and after pid recycling on a busy host `killpg` + # would target an unrelated group. In the sudo branch the pid is + # not even the right identifier, since it belongs to `sudo` and + # not to the workload's process group. + # + # None is the established "unknown" value on this field -- + # find_sudo_child_process_group_id returns it for the same reason + # (_linux/_sudo.py), and _posix_signal_subprocess already retries + # the lookup and then declines to signal when it is still unset. + self._sudo_child_process_group_id = None + self._logger.debug( + f"Process {self._process.pid} exited before its process group could be " + "determined; no process group will be signalled for it.", + extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), ) self._logger.info( @@ -202,11 +241,81 @@ def run(self) -> None: if self._callback: self._callback() finally: - # Explicitly delete the Popen in case it's a PopenWindowsAsUser and there's stuff to - # deallocate + # Drop this object's reference to the Popen, in case it's a + # PopenWindowsAsUser and there's stuff to deallocate. `proc` itself is + # a function local and dies with the frame; a `del proc` here would be + # a no-op, which is what CodeQL's "unnecessary delete" reports. proc = self._process self._process = None - del proc + # This method owns the subprocess for its whole life, and clearing + # `_process` is the point after which nothing else can reach it: + # `notify()` and `terminate()` both become permanent no-ops. So + # ownership has to be discharged here, on every exit path, not only on + # the one that runs `wait()` above. + # + # An exception out of the stdout pump used to skip both the `wait()` + # and the returncode capture while still clearing `_process` -- + # leaving a live, unreapable child, no exit code, and an object + # reporting neither running nor failed-to-start. A raising + # `logging.Filter` reaches this path, and installing one is a + # supported use of this library. + self._reap(proc) + + def _reap(self, proc: Optional[Popen]) -> None: + """Make sure ``proc`` is dead and reaped, and that its exit code is + recorded, however :meth:`run` is leaving. + + A no-op on the normal path, where ``run`` has already waited and captured + the returncode. + """ + if proc is None: + return + if proc.poll() is None: + # Still running, and we are on our way out: nothing will be able to + # signal it once `_process` is cleared. Terminating is the lesser + # evil -- the alternative is an orphan holding the session directory + # and, for a cross-user action, running as the job user with no + # owner. + self._logger.error( + f"Abandoning the subprocess {proc.pid} before it exited; terminating it so it " + "is not orphaned.", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO + ), + ) + try: + self._terminate_process(proc) + except Exception as e: # noqa: BLE001 + self._logger.error( + f"Could not terminate the abandoned subprocess {proc.pid}: {e}", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO + ), + ) + try: + # Bounded: a SIGKILLed child is reaped promptly, and this runs on + # the pool worker, so an unbounded wait here would hang the + # runner's future rather than surface anything useful. + proc.wait(timeout=ABANDONED_PROCESS_REAP_TIMEOUT_SECONDS) + except TimeoutExpired: + self._logger.error( + f"Subprocess {proc.pid} did not exit within " + f"{ABANDONED_PROCESS_REAP_TIMEOUT_SECONDS}s of being terminated; it is left " + "unreaped.", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO + ), + ) + # Record the exit status. Unconditional on purpose: `Popen.returncode` is + # populated by the poll()/wait() above and is the authority either way -- + # on the normal path `run` has already assigned the same value, and on the + # abandoned path this is the only assignment. It is None only if the child + # outlived the terminate timeout, where "unknown" is the honest answer. + # + # An `if self._returncode is None:` guard here would read as defensive but + # could never change the outcome, and a mutation test proved it: removing + # it left every test passing. An unfalsifiable check is worse than none. + self._returncode = proc.returncode def notify(self) -> None: """The 'Notify' part of Open Job Description's subprocess cancelation method. @@ -218,11 +327,17 @@ def notify(self) -> None: TODO: Send the signal to every direct and transitive child of the parent process. """ - if self._process is not None and self._process.poll() is None: + # Review22-F4 fix: Bind _process once. Double-load is a TOCTOU race: + # the None check and poll() call read _process separately, allowing + # another thread to set it to None between them. + proc = self._process + if proc is not None and proc.poll() is None: if is_posix(): - self._posix_signal_subprocess(signal_name="term") + # R4-G8 fix: Pass the bound proc to helpers to avoid reloading + # self._process, which could be None by the time the helper runs. + self._posix_signal_subprocess(proc, signal_name="term") else: - self._windows_notify_subprocess() + self._windows_notify_subprocess(proc) def terminate(self) -> None: """The 'Terminate' part of Open Job Description's subprocess cancelation method. @@ -234,15 +349,26 @@ def terminate(self) -> None: TODO: Send the signal to every direct and transitive child of the parent process. """ - if self._process is not None and self._process.poll() is None: - if is_posix(): - self._posix_signal_subprocess(signal_name="kill") - else: - self._logger.info( - f"INTERRUPT: Start killing the process tree with the root pid: {self._process.pid}", - extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), - ) - kill_windows_process_tree(self._logger, self._process.pid, signal_subprocesses=True) + # Review22-F4 fix: Bind _process once. See notify() for rationale. + proc = self._process + if proc is not None and proc.poll() is None: + self._terminate_process(proc) + + def _terminate_process(self, process: Popen) -> None: + """Deliver the terminate signal to ``process``. + + Split out of :meth:`terminate` so that :meth:`run`'s ``finally`` can reach + it after ``self._process`` has been cleared -- it takes the Popen as an + argument for the same reason the platform helpers do (R4-G8). + """ + if is_posix(): + self._posix_signal_subprocess(process, signal_name="kill") + else: + self._logger.info( + f"INTERRUPT: Start killing the process tree with the root pid: {process.pid}", + extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), + ) + kill_windows_process_tree(self._logger, process.pid, signal_subprocesses=True) def _start_subprocess(self) -> Optional[Popen]: """Helper invoked by self.run() to start up the subprocess.""" @@ -347,10 +473,19 @@ def _log_subproc_stdout(self) -> None: Then the thread exits and is cleaned up automatically by the python garbage collector. """ - assert self._process - stream = self._process.stdout - # Convince type checker that stdout is not None - assert stream is not None + # R5-6: explicit raises rather than `assert`. This method owns the + # subprocess's output stream for the whole life of the child, and it runs + # on the pool worker; stripping these under `python -O` would turn a + # legible "called with no process" into an AttributeError on None from + # inside the pump loop. + process = self._process + if process is None: # pragma: no cover - defensive + raise RuntimeError( + "Internal error: cannot forward stdout before the subprocess has been created." + ) + stream = process.stdout + if stream is None: # pragma: no cover - defensive + raise RuntimeError("Internal error: the subprocess was created without a stdout pipe.") exit_event = Event() @@ -404,7 +539,7 @@ def _enqueue_stdout(): except Empty: pass # queue.get timed out. This means the subprocess does not print much to STDOUT. Just continue. - if self._process.poll() is not None: # The main command exited. + if process.poll() is not None: # The main command exited. if process_exit_time is None: process_exit_time = time.monotonic() elif (time.monotonic() - process_exit_time) < 1: @@ -455,21 +590,21 @@ def _tosigned(n: int) -> int: def _posix_signal_subprocess( self, + process: Popen, signal_name: Literal["term", "kill"], ) -> None: - """Send a given named signal to the subprocess.""" + """Send a given named signal to the subprocess. + + Args: + process: The Popen object to signal. Passed from caller to avoid + reloading self._process, which could be None by now (R4-G8 fix). + signal_name: Either "term" for SIGTERM or "kill" for SIGKILL. + """ # Hint to mypy to not raise module attribute errors (e.g. missing os.getpgid) if sys.platform == "win32": raise NotImplementedError("This method is for POSIX hosts only") - # We can run into a race condition where the process exits (and another thread sets self._process to None) - # before the cancellation happens, so we swap to a local variable to ensure a cancellation that is not needed, - # does not raise an exception here. - process = self._process - # Convince the type checker that accessing process is okay - assert process is not None - # Note: A limitation of this implementation is that it will only sigkill # processes that are in the same process-group as the command that we ran. # In the future, we can extend this to killing all processes spawned (including into @@ -599,11 +734,13 @@ def _log_process_tree(self) -> None: extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), ) - def _windows_notify_subprocess(self) -> None: - """Sends a CTRL_BREAK_EVENT signal to the subprocess""" - # Convince the type checker that accessing _process is okay - assert self._process is not None + def _windows_notify_subprocess(self, process: Popen) -> None: + """Sends a CTRL_BREAK_EVENT signal to the subprocess. + Args: + process: The Popen object to signal. Passed from caller to avoid + reloading self._process, which could be None by now (R4-G8 fix). + """ # CTRL-C handler is disabled by default when CREATE_NEW_PROCESS_GROUP is passed. # We send CTRL-BREAK as handler for it cannnot be disabled. # https://learn.microsoft.com/en-us/windows/console/ctrl-c-and-ctrl-break-signals @@ -611,18 +748,18 @@ def _windows_notify_subprocess(self) -> None: # https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessa#remarks # https://stackoverflow.com/questions/35772001/how-to-handle-a-signal-sigint-on-a-windows-os-machine/35792192#35792192 self._logger.info( - f"INTERRUPT: Sending CTRL_BREAK_EVENT to {self._process.pid}", + f"INTERRUPT: Sending CTRL_BREAK_EVENT to {process.pid}", extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), ) - # _process will be running in new console, we run another process to attach to it and send signal + # process will be running in new console, we run another process to attach to it and send signal cmd = [ # When running in a service context, we want to call the non-service Python binary sys.executable.lower().replace("pythonservice.exe", "python.exe"), str(WINDOWS_SIGNAL_SUBPROC_SCRIPT_PATH), - str(self._process.pid), + str(process.pid), ] - process = LoggingSubprocess( + signal_subprocess = LoggingSubprocess( logger=self._logger, args=cmd, encoding=self._encoding, @@ -633,11 +770,11 @@ def _windows_notify_subprocess(self) -> None: ) # Blocking call - process.run() + signal_subprocess.run() - if process.exit_code != 0: + if signal_subprocess.exit_code != 0: self._logger.warning( - f"Failed to send signal 'CTRL_BREAK_EVENT' to subprocess {self._process.pid}", + f"Failed to send signal 'CTRL_BREAK_EVENT' to subprocess {process.pid}", extra=LogExtraInfo( openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO ), diff --git a/src/openjd/sessions/_tempdir.py b/src/openjd/sessions/_tempdir.py index 60afc2bc..c96f9702 100644 --- a/src/openjd/sessions/_tempdir.py +++ b/src/openjd/sessions/_tempdir.py @@ -2,11 +2,13 @@ import os import stat +import sys from ._logging import LoggerAdapter, LogContent, LogExtraInfo from pathlib import Path -from shutil import chown, rmtree +from shutil import rmtree +from ._embedded_files import chown_group from tempfile import gettempdir, mkdtemp -from typing import Optional, cast +from typing import Any, Optional, cast from ._session_user import PosixSessionUser, SessionUser, WindowsSessionUser from ._windows_permission_helper import WindowsPermissionHelper @@ -16,6 +18,150 @@ from ._win32._helpers import get_process_user # type: ignore +OPENJD_TEMPDIR_MODE = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH +"""Mode for the shared Open Job Description temporary root: 0o755. + +World-traversable on purpose. If this directory lacked group/other search +permission then a job user -- who by design is a different OS user from this +process -- could not reach the session working directory beneath it. The session +directories themselves are created 0o700 by mkdtemp() and only widened to 0o770 +for the specific group named by a SessionUser, so the permissive mode here grants +traversal, not read access to any session's contents. + +Applied explicitly rather than left to the process umask so that the resulting +mode does not depend on how the embedding application happens to be configured. +""" + + +def _prepare_temp_dir_root_posix(temp_dir: str) -> None: + """Validate the shared temporary root and set its mode, operating on a single + file descriptor so the two cannot disagree. + + Why this exists: the root is a *fixed, predictable* path + (``/OpenJD``) and on POSIX its parent is typically world-writable + ``/tmp``. ``os.makedirs(..., exist_ok=True)`` happily accepts an entry that + some other local user created first, or a symlink pointing somewhere else + entirely -- and every session working directory would then be created inside + a directory that user controls. ``/tmp``'s sticky bit does not help: it + restricts deletion within ``/tmp`` itself, not within an attacker-owned + ``/tmp/OpenJD``. + + Why a descriptor rather than the path: an earlier version of this validated + with ``os.lstat(path)`` and then called ``os.stat(path)``/``os.chmod(path)``. + Both of those resolve the *name* again and follow symlinks, so replacing the + entry with a symlink in between defeated the check entirely -- and the chmod + then widened the link's target to 0o755. Opening once with ``O_NOFOLLOW`` and + ``O_DIRECTORY`` and using ``fstat``/``fchmod`` means every decision and every + modification applies to the same inode we validated, whatever happens to the + name afterwards. + + ``O_NOFOLLOW`` makes the open itself fail on a symlink, which is why there is + no separate ``S_ISLNK`` branch. + + POSIX only. Windows cannot ``os.open()`` a directory at all -- the CRT + rejects it with ``EACCES`` regardless of flags -- so Windows takes the + ``lstat`` path in :func:`_prepare_temp_dir_root_windows` instead. + + Raises: + RuntimeError: if the path is a symlink, is not a directory, is owned by + another user, or its mode cannot be set. + """ + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_DIRECTORY # type: ignore + try: + fd = os.open(temp_dir, flags) + except OSError as err: + # ELOOP here is the O_NOFOLLOW refusal; ENOTDIR is O_DIRECTORY's. + raise RuntimeError( + f"Refusing to use temporary directory {temp_dir}: it could not be opened as a real " + f"directory ({err}). If it is a symbolic link or not a directory, remove it, or pass " + "an explicit session root directory." + ) + try: + st = os.fstat(fd) + if not stat.S_ISDIR(st.st_mode): + raise RuntimeError( + f"Refusing to use temporary directory {temp_dir}: it is not a real directory." + ) + this_uid = os.geteuid() # type: ignore + # root is accepted so that a system-provisioned root directory works. + if st.st_uid not in (this_uid, 0): + raise RuntimeError( + f"Refusing to use temporary directory {temp_dir}: it is owned by uid " + f"{st.st_uid}, not by this process' uid ({this_uid}) or root. Another user " + "may have created it. Remove it, or pass an explicit session root directory." + ) + + # makedirs()' `mode` is masked by the process umask, so it does not on its + # own guarantee anything: under umask 0o077 an "explicit" 0o755 lands as + # 0o700, and a job user could then not traverse into its own session + # directory. Set the mode outright. Conditional because a + # system-provisioned root may be owned by root with the right mode + # already, where the call would fail for no reason. + if stat.S_IMODE(st.st_mode) != stat.S_IMODE(OPENJD_TEMPDIR_MODE): + try: + # fchmod, not chmod: applies to the validated inode, not to + # whatever the name resolves to now. + os.fchmod(fd, OPENJD_TEMPDIR_MODE) # type: ignore + except OSError as err: + raise RuntimeError( + f"Could not set permissions on temporary directory {temp_dir}: {err}. " + "Session working directories created beneath it may not be reachable by a " + "job user." + ) + finally: + os.close(fd) + + +def _prepare_temp_dir_root_windows(temp_dir: str) -> None: + """Validate the shared temporary root on Windows. + + Windows cannot open a directory with ``os.open()`` -- the CRT returns + ``EACCES`` whatever the flags -- so the single-descriptor approach used on + POSIX is not available here. ``os.lstat()`` is the closest equivalent: it + does not traverse symlinks or reparse points, so a directory symlink reports + ``S_ISLNK`` (failing the ``S_ISDIR`` test) and a junction reports a non-zero + ``st_reparse_tag``. + + There is no mode to set: ``OPENJD_TEMPDIR_MODE`` is a POSIX mode, and + Windows access to this root is governed by the ACLs it inherits from + ``%PROGRAMDATA%``. There is also no uid comparison; ownership here is a SID, + and asserting anything about it is left to the Windows-specific code that + creates session directories beneath this root. + + This is a weaker check than the POSIX one, and knowingly so: it closes the + "the path is not really a directory" case but not a time-of-check race. + + Raises: + RuntimeError: if the path is a symlink, reparse point, or not a directory. + """ + try: + st = os.lstat(temp_dir) + except OSError as err: + raise RuntimeError( + f"Refusing to use temporary directory {temp_dir}: it could not be inspected ({err}). " + "Remove it, or pass an explicit session root directory." + ) + if not stat.S_ISDIR(st.st_mode) or getattr(st, "st_reparse_tag", 0) != 0: + raise RuntimeError( + f"Refusing to use temporary directory {temp_dir}: it is not a real directory. If it " + "is a symbolic link, junction, or not a directory, remove it, or pass an explicit " + "session root directory." + ) + + +def _prepare_temp_dir_root(temp_dir: str) -> None: + """Validate the shared temporary root, by whichever means the platform allows. + + See :func:`_prepare_temp_dir_root_posix` and + :func:`_prepare_temp_dir_root_windows`; the two differ in strength, not just + in mechanism. + """ + if is_posix(): + _prepare_temp_dir_root_posix(temp_dir) + else: + _prepare_temp_dir_root_windows(temp_dir) + + def custom_gettempdir(logger: Optional[LoggerAdapter] = None) -> str: """ Get a platform-specific temporary directory. @@ -30,6 +176,11 @@ def custom_gettempdir(logger: Optional[LoggerAdapter] = None) -> str: Returns: str: The path to the temporary directory specific to the operating system. + + Raises: + RuntimeError: If the directory could not be created, or if it already + exists and is not a directory owned by this process' user (see + :func:`_prepare_temp_dir_root`). """ if is_windows(): program_data_path = os.getenv("PROGRAMDATA") @@ -46,7 +197,16 @@ def custom_gettempdir(logger: Optional[LoggerAdapter] = None) -> str: temp_dir_parent = gettempdir() temp_dir = os.path.join(temp_dir_parent, "OpenJD") - os.makedirs(temp_dir, exist_ok=True) + try: + os.makedirs(temp_dir, mode=OPENJD_TEMPDIR_MODE, exist_ok=True) + except OSError as err: + raise RuntimeError(f"Could not create temporary directory {temp_dir}: {err}") + + # R5-3 fix: exist_ok=True accepts whatever is already at this predictable + # path -- another local user's directory, or a symlink pointing elsewhere. + # Validate and set the mode through one descriptor, so the entry cannot be + # swapped between the two. + _prepare_temp_dir_root(temp_dir) return temp_dir @@ -60,6 +220,28 @@ class TempDir: do it (don't really need to, either, since the use-case for this class is to create the Open Job Description Session working directory and that working directory needs to be both writable and deletable by this process). + + Trust precondition when ``user`` is given (POSIX): + ``mkdtemp()``'s 0o700 is deliberately widened to 0o770 so that the target + user can write into the session directory. That grant is to a *group*, + not to a user, so **every member of the group named by the + ``PosixSessionUser`` gains write access to the whole session tree** -- + not only the intended session user. Generated shell scripts and + materialized embedded files live in that tree and are executed after + being written, so a second member of that group could substitute their + contents between materialization and exec. + + The caller is therefore required to supply a group that contains only + principals it already trusts with the session's work -- conventionally a + group dedicated to the pairing of this process' user and the one job + user. This runtime cannot verify that and does not try to. The ordering + here is careful about the part it can control: group ownership is changed + *before* the mode is widened, so a failure to set the group never leaves + a group-writable directory behind. + + The same precondition covers the group-writable bits that + ``_embedded_files.write_file_for_user`` sets on individual files for the + same reason. """ path: Path @@ -105,51 +287,86 @@ def __init__( except OSError as err: raise RuntimeError(f"Could not create temp directory within {str(dir)}: {str(err)}") - # Change the owner - if user: - if is_posix(): - user = cast(PosixSessionUser, user) - # Change ownership - try: - chown(self.path, group=user.group) - except OSError as err: - raise RuntimeError( - f"Could not change ownership of directory '{str(dir)}' (error: {str(err)}). Please ensure that uid {os.geteuid()} is a member of group {user.group}." # type: ignore - ) - # Update the permissions to include the group after the group is changed - # Note: Only after changing group for security in case the group-ownership - # change fails. - os.chmod(self.path, mode=stat.S_IRWXU | stat.S_IRWXG) - elif is_windows(): - user = cast(WindowsSessionUser, user) - try: - WindowsPermissionHelper.set_permissions( - str(self.path), - principals_full_control=[get_process_user()], - principals_modify_access=[user.user], - ) - except Exception as err: - raise RuntimeError( - f"Could not change permissions of directory '{str(dir)}' (error: {str(err)})" - ) + # Change the owner. + # + # Wrapped so that a failure here does not leave the mkdtemp directory + # behind: construction has failed, so the caller has no handle to clean up + # with, and this directory sits under a shared root that nothing else ever + # prunes. Reachable from a mis-typed `PosixSessionUser.group` -- which is + # not validated anywhere -- and not only from a permissions problem. + try: + if user: + if is_posix(): + user = cast(PosixSessionUser, user) + # Change ownership + try: + # chown_group, not shutil.chown: an unresolvable group name + # raises LookupError, which this handler would not catch. + chown_group(self.path, user.group) + except OSError as err: + raise RuntimeError( + f"Could not change ownership of directory '{str(dir)}' (error: {str(err)}). Please ensure that uid {os.geteuid()} is a member of group {user.group}." # type: ignore + ) + # Update the permissions to include the group after the group is changed + # Note: Only after changing group for security in case the group-ownership + # change fails. + os.chmod(self.path, mode=stat.S_IRWXU | stat.S_IRWXG) + elif is_windows(): + user = cast(WindowsSessionUser, user) + try: + WindowsPermissionHelper.set_permissions( + str(self.path), + principals_full_control=[get_process_user()], + principals_modify_access=[user.user], + ) + except Exception as err: + raise RuntimeError( + f"Could not change permissions of directory '{str(dir)}' (error: {str(err)})" + ) + except BaseException: + # Best-effort: the failure being reported is the interesting one, so a + # cleanup problem must not replace it. + rmtree(self.path, ignore_errors=True) + raise def cleanup(self) -> None: """Deletes the temporary directory and all of its contents. Raises: - RuntimeError - If not all files could be deleted. + RuntimeError - If not all files could be deleted. The message names + each path that could not be deleted along with the reason. """ - encountered_errors = False - file_paths: list[str] = [] + failures: list[str] = [] + + def _record(path: Any, error: BaseException) -> None: + # R5-7 fix: keep the reason. Cleanup is exactly where the difference + # between "permission denied" and "a process still holds this open" + # decides what the operator should do next, and the old handler + # accepted the exception and discarded it -- leaving a list of bare + # paths with no cause. + failures.append(f"{path}: {error!r}") + + if sys.version_info >= (3, 12): + # `onerror` is deprecated from 3.12 and slated for removal; `onexc` + # receives the exception instance directly. + def onexc( + func: Any, path: Any, error: BaseException + ) -> None: # pragma: nocover - version-gated + _record(path, error) + + rmtree(self.path, onexc=onexc) + else: + + def onerror( + func: Any, path: Any, exc_info: Any + ) -> None: # pragma: nocover - version-gated + # Pre-3.12 handlers are called with sys.exc_info()-style triples. + error = exc_info[1] if isinstance(exc_info, tuple) else exc_info + _record(path, error) - def onerror(f, p, e): - nonlocal encountered_errors - nonlocal file_paths - encountered_errors = True - file_paths.append(str(p)) + rmtree(self.path, onerror=onerror) - rmtree(self.path, onerror=onerror) - if encountered_errors: + if failures: raise RuntimeError( f"Files within temporary directory {str(self.path)} could not be deleted.\n" - + "\n".join(file_paths) + + "\n".join(failures) ) diff --git a/src/openjd/sessions/_types.py b/src/openjd/sessions/_types.py index db62db9d..310a766e 100644 --- a/src/openjd/sessions/_types.py +++ b/src/openjd/sessions/_types.py @@ -20,6 +20,14 @@ EnvironmentModel = Environment_2023_09 EnvironmentScriptModel = EnvironmentScript_2023_09 +# Default notifyPeriodInSeconds for a NOTIFY_THEN_TERMINATE cancelation +# when the action omits the field (2023-09 Template Schemas 5.3.2). +TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS = 120 +"""Default notify period for a Step Script's onRun action.""" +ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS = 30 +"""Default notify period for any other action (e.g. an Environment's +onEnter/onExit).""" + class ActionState(str, Enum): RUNNING = "running" diff --git a/src/openjd/sessions/_v1/_linux/__init__.py b/src/openjd/sessions/_v1/_linux/__init__.py deleted file mode 100644 index 8d929cc8..00000000 --- a/src/openjd/sessions/_v1/_linux/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/src/openjd/sessions/_v1/_linux/_capabilities.py b/src/openjd/sessions/_v1/_linux/_capabilities.py deleted file mode 100644 index c2c31a58..00000000 --- a/src/openjd/sessions/_v1/_linux/_capabilities.py +++ /dev/null @@ -1,257 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - -"""This module contains code for interacting with Linux capabilities. The module uses the ctypes -module from the Python standard library to wrap the libcap library. - -See https://man7.org/linux/man-pages/man7/capabilities.7.html for details on this Linux kernel -feature. -""" - -import ctypes -import os -import sys -from contextlib import contextmanager -from ctypes.util import find_library -from enum import Enum -from functools import cache -from typing import Any, Generator, Optional, Tuple, TYPE_CHECKING - - -from .._logging import LOG - -# Capability sets -CAP_EFFECTIVE = 0 -CAP_PERMITTED = 1 -CAP_INHERITABLE = 2 - -# Capability bit numbers -CAP_KILL = 5 - -# Values for cap_flag_value_t arguments -CAP_CLEAR = 0 -CAP_SET = 1 - -cap_flag_t = ctypes.c_int -cap_flag_value_t = ctypes.c_int -cap_value_t = ctypes.c_int - - -class CapabilitySetType(Enum): - INHERITABLE = CAP_INHERITABLE - PERMITTED = CAP_PERMITTED - EFFECTIVE = CAP_EFFECTIVE - - -class UserCapHeader(ctypes.Structure): - _fields_ = [ - ("version", ctypes.c_uint32), - ("pid", ctypes.c_int), - ] - - -class UserCapData(ctypes.Structure): - _fields_ = [ - ("effective", ctypes.c_uint32), - ("permitted", ctypes.c_uint32), - ("inheritable", ctypes.c_uint32), - ] - - -class Cap(ctypes.Structure): - _fields_ = [ - ("head", UserCapHeader), - ("data", UserCapData), - ] - - -if TYPE_CHECKING: - cap_t = ctypes._Pointer[Cap] - cap_flag_value_ptr = ctypes._Pointer[cap_flag_value_t] - cap_value_ptr = ctypes._Pointer[cap_value_t] - ssize_ptr_t = ctypes._Pointer[ctypes.c_ssize_t] -else: - cap_t = ctypes.POINTER(Cap) - cap_flag_value_ptr = ctypes.POINTER(cap_flag_value_t) - cap_value_ptr = ctypes.POINTER(cap_value_t) - ssize_ptr_t = ctypes.POINTER(ctypes.c_ssize_t) - - -def _cap_set_err_check( - result: ctypes.c_int, - func: Any, - args: Tuple[Any, ...], -) -> ctypes.c_int: - if result != 0: - errno = ctypes.get_errno() - raise OSError(errno, os.strerror(errno)) - return result - - -def _cap_get_proc_err_check( - result: cap_t, - func: Any, - args: Tuple[cap_t, cap_flag_t, ctypes.c_int, cap_value_ptr, cap_flag_value_t], -) -> cap_t: - if not result: - errno = ctypes.get_errno() - raise OSError(errno, os.strerror(errno)) - return result - - -def _cap_get_flag_errcheck( - result: ctypes.c_int, func: Any, args: Tuple[cap_t, cap_value_t, cap_flag_t, cap_flag_value_ptr] -) -> ctypes.c_int: - if result != 0: - errno = ctypes.get_errno() - raise OSError(errno, os.strerror(errno)) - return result - - -@cache -def _get_libcap() -> Optional[ctypes.CDLL]: - if not sys.platform.startswith("linux"): - raise OSError(f"libcap is only available on Linux, but found platform: {sys.platform}") - - libcap_path = find_library("cap") - if libcap_path is None: - LOG.info( - "Unable to locate libcap. Session action cancelation signals will be sent using sudo" - ) - return None - - libcap = ctypes.CDLL(libcap_path, use_errno=True) - - # https://man7.org/linux/man-pages/man3/cap_set_proc.3.html - libcap.cap_set_proc.restype = ctypes.c_int - libcap.cap_set_proc.argtypes = [ - cap_t, - ] - libcap.cap_set_proc.errcheck = _cap_set_err_check # type: ignore - - # https://man7.org/linux/man-pages/man3/cap_get_proc.3.html - libcap.cap_get_proc.restype = cap_t - libcap.cap_get_proc.argtypes = [] - libcap.cap_get_proc.errcheck = _cap_get_proc_err_check # type: ignore - - # https://man7.org/linux/man-pages/man3/cap_set_flag.3.html - libcap.cap_set_flag.restype = ctypes.c_int - libcap.cap_set_flag.argtypes = [ - cap_t, - cap_flag_t, - ctypes.c_int, - cap_value_ptr, - cap_flag_value_t, - ] - - # https://man7.org/linux/man-pages/man3/cap_get_flag.3.html - libcap.cap_get_flag.restype = ctypes.c_int - libcap.cap_get_flag.argtypes = ( - cap_t, - cap_value_t, - cap_flag_t, - cap_flag_value_ptr, - ) - libcap.cap_get_flag.errcheck = _cap_get_flag_errcheck # type: ignore - - return libcap - - -def _has_capability( - *, - libcap: ctypes.CDLL, - caps: cap_t, - capability: int, - capability_set_type: CapabilitySetType, -) -> bool: - flag_value = cap_flag_value_t() - libcap.cap_get_flag(caps, capability, capability_set_type.value, ctypes.byref(flag_value)) - return flag_value.value == CAP_SET - - -@contextmanager -def try_use_cap_kill() -> Generator[bool, None, None]: - """ - A context-manager that attempts to leverage the CAP_KILL Linux capability. - - If CAP_KILL is in the current thread's effective set, this context-manager takes no action and - yields True. - - If CAP_KILL is not in the effective set but is in the permitted set, the context-manager: - 1. adds CAP_KILL to the effective set before entering the context-manager - 2. yields True - 3. clears CAP_KILL from the effective set when exiting the context-manager - - Otherwise, the context-manager does nothing and yields False - - Returns: - A context manager that yields a bool. See above for details. - """ - if not sys.platform.startswith("linux"): - raise OSError(f"Only Linux is supported, but platform is {sys.platform}") - - libcap = _get_libcap() - # If libcap is not found, we yield False indicating we are not aware of having CAP_KILL - if not libcap: - yield False - return - - caps = libcap.cap_get_proc() - - if _has_capability( - libcap=libcap, - caps=caps, - capability=CAP_KILL, - capability_set_type=CapabilitySetType.EFFECTIVE, - ): - LOG.debug("CAP_KILL is in the thread's effective set") - # CAP_KILL is already in the effective set - yield True - elif _has_capability( - libcap=libcap, - caps=caps, - capability=CAP_KILL, - capability_set_type=CapabilitySetType.PERMITTED, - ): - # CAP_KILL is in the permitted set. We will temporarily add it to the effective set - LOG.debug("CAP_KILL is in the thread's permitted set. Temporarily adding to effective set") - cap_value_arr_t = cap_value_t * 1 - cap_value_arr = cap_value_arr_t() - cap_value_arr[0] = CAP_KILL - libcap.cap_set_flag( - caps, - CAP_EFFECTIVE, - 1, - cap_value_arr, - CAP_SET, - ) - libcap.cap_set_proc(caps) - try: - yield True - finally: - # Clear CAP_KILL from the effective set - LOG.debug("Clearing CAP_KILL from the thread's effective set") - libcap.cap_set_flag( - caps, - CAP_EFFECTIVE, - 1, - cap_value_arr, - CAP_CLEAR, - ) - libcap.cap_set_proc(caps) - else: - yield False - - -def main() -> None: - """A developer debugging entrypoint for testing the try_use_cap_kill() behaviour""" - import logging - - logging.basicConfig(level=logging.DEBUG) - logging.getLogger("openjd.sessions").setLevel(logging.DEBUG) - - with try_use_cap_kill() as has_cap_kill: - LOG.info("Has CAP_KILL: %s", has_cap_kill) - - -if __name__ == "__main__": - main() diff --git a/src/openjd/sessions/_v1/_linux/_sudo.py b/src/openjd/sessions/_v1/_linux/_sudo.py deleted file mode 100644 index 43a1fa8c..00000000 --- a/src/openjd/sessions/_v1/_linux/_sudo.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - -import glob -import os -import sys -import time -from subprocess import Popen, DEVNULL, PIPE, STDOUT, run -from typing import Optional - -from .._logging import LoggerAdapter, LogContent, LogExtraInfo -from .._os_checker import is_posix, is_linux - - -class FindSignalTargetError(Exception): - """Exception when unable to detect the signal target""" - - pass - - -def find_sudo_child_process_group_id( - *, - logger: LoggerAdapter, - sudo_process: Popen, - timeout_seconds: float = 1, -) -> Optional[int]: - # Hint to mypy to not raise module attribute errors (e.g. missing os.getpgid) - if sys.platform == "win32": - raise NotImplementedError("This method is for POSIX hosts only") - if not is_posix(): - raise NotImplementedError(f"Only POSIX supported, but running on {sys.platform}") - if timeout_seconds <= 0: - raise ValueError(f"Expected positive value for timeout_seconds but got {timeout_seconds}") - - # For cross-user support, we use sudo which creates an intermediate process: - # - # openjd-process - # | - # +-- sudo - # | - # +-- subprocess - # - # Sudo forwards signals that it is able to handle, but in the case of SIGKILL sudo cannot - # handle the signal and the kernel will kill it leaving the child orphaned. We need to - # send SIGKILL signals to the subprocess of sudo - start = time.monotonic() - now = start - sudo_pgid = os.getpgid(sudo_process.pid) - - # Repeatedly scan for child processes - # - # This is put in a retry loop, because it takes a non-zero amount of time before sudo and - # the kernel finish creating the subprocess. We cap this because the process may exit - # quickly and we may never find the child process. - sudo_child_pid: Optional[int] = None - sudo_child_pgid: Optional[int] = None - try: - while now - start < timeout_seconds: - if not sudo_child_pid: - if is_linux(): - sudo_child_pid = find_sudo_child_process_id_procfs( - sudo_pid=sudo_process.pid, - logger=logger, - ) - else: - sudo_child_pid = find_child_process_id_pgrep( - sudo_pid=sudo_process.pid, - ) - - if sudo_child_pid: - try: - sudo_child_pgid = os.getpgid(sudo_child_pid) - except ProcessLookupError: - # If the process has exited, we short-circuit - return None - # sudo first forks, then creates a new process group. There is a race condition - # where the process group ID we observe has not yet changed. If the PGID detected - # matches the PGID of sudo, then we retry again in the loop - if sudo_child_pgid == sudo_pgid: - sudo_child_pgid = None - else: - break - - # If we did not find any child processes yet, sleep for some time and retry - time.sleep(min(0.05, timeout_seconds - (now - start))) - now = time.monotonic() - if not sudo_child_pid or not sudo_child_pgid: - raise FindSignalTargetError("unable to detect subprocess before timeout") - except FindSignalTargetError as e: - logger.warning( - f"Unable to determine signal target: {e}", - extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), - ) - - if sudo_child_pgid: - logger.debug( - f"Signal target PGID = {sudo_child_pgid}", - extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), - ) - - return sudo_child_pgid - - -def find_sudo_child_process_id_procfs( - *, - logger: LoggerAdapter, - sudo_pid: int, -) -> Optional[int]: - # Look for the child process of sudo using procfs. See - # https://docs.kernel.org/filesystems/proc.html#proc-pid-task-tid-children-information-about-task-children - - child_pids: set[int] = set() - for task_children_path in glob.glob(f"/proc/{sudo_pid}/task/**/children"): - with open(task_children_path, "r") as f: - child_pids.update(int(pid_str) for pid_str in f.read().split()) - - # If we found exactly one child, we return it - if len(child_pids) == 1: - - child_pid = child_pids.pop() - - logger.debug( - f"Session action process (sudo child) PID is {child_pid}", - extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), - ) - return child_pid - # If we found multiple child processes, this violates our assumptions about how sudo - # works. We will fall-back to using pkill for signalling the process - elif len(child_pids) > 1: - raise FindSignalTargetError( - f"Expected single child processes of sudo, but found {child_pids}" - ) - return None - - -def find_child_process_id_pgrep( - *, - sudo_pid: int, -) -> Optional[int]: - pgrep_result = run( - ["pgrep", "-P", str(sudo_pid)], - stdout=PIPE, - stderr=STDOUT, - stdin=DEVNULL, - text=True, - ) - if pgrep_result.returncode != 0: - raise FindSignalTargetError("Unable to query child processes of sudo process") - results = pgrep_result.stdout.splitlines() - if len(results) > 1: - raise FindSignalTargetError(f"Expected a single child process of sudo, but found {results}") - elif len(results) == 0: - return None - sudo_subproc_pid = int(results[0]) - return sudo_subproc_pid diff --git a/src/openjd/sessions/_windows_process_killer.py b/src/openjd/sessions/_windows_process_killer.py index 2809ce99..75856c75 100644 --- a/src/openjd/sessions/_windows_process_killer.py +++ b/src/openjd/sessions/_windows_process_killer.py @@ -60,8 +60,21 @@ def _suspend_process_tree( if not suspend_subprocesses: return - # Recursively suspend child processes. - for child in process.children(): + # Recursively suspend child processes. A process that exited between being + # discovered and being walked has no children to suspend, and must not fail + # the cancel: psutil raises NoSuchProcess ("process PID not found") here, and + # this runs on the run future, so the exception would surface as a failed + # action for a subprocess that in fact completed. + try: + children = process.children() + except NoSuchProcess: + logger.info( + f"Process {process.pid} exited before its children could be listed.", + extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), + ) + return + + for child in children: _suspend_process_tree( logger, child, all_processes, procs_cannot_suspend, suspend_subprocesses ) diff --git a/test/openjd/sessions_v0/conftest.py b/test/openjd/sessions_v0/conftest.py index eca66db8..db227f3e 100644 --- a/test/openjd/sessions_v0/conftest.py +++ b/test/openjd/sessions_v0/conftest.py @@ -3,6 +3,8 @@ import os import random import string +import time +import uuid from logging import INFO, getLogger from logging.handlers import QueueHandler from queue import Empty, SimpleQueue @@ -18,6 +20,9 @@ from openjd.sessions._action_filter import ActionMonitoringFilter from openjd.model import RevisionExtensions, SpecificationRevision +if is_posix(): + import grp + if is_windows(): from openjd.sessions._win32._helpers import ( # type: ignore get_current_process_session_id, @@ -62,6 +67,107 @@ def pytest_collection_modifyitems(config, items): config.option.markexpr = mark_expr +SERIAL_PROCESS_GROUP = "serial_process" + +serial_process = pytest.mark.xdist_group(SERIAL_PROCESS_GROUP) +"""Mark for tests that race a real subprocess against wall-clock expectations. + +Applied to a class or a test, it pins every such test onto ONE xdist worker, so +they run serially with respect to each other instead of competing for CPU with +eleven siblings that are each also sleeping on a child process. + +Why this is needed: these tests start a child, cancel or time it out, and assert +on the outcome. The assertions are correct, but they assume the child and the +runtime get scheduled reasonably promptly. Under `-n auto` on a loaded host that +assumption fails, and the whole cancel/terminate family goes red together while +the product is behaving correctly -- observed as 14 simultaneous failures in a +108-second run that all passed serially. The failure mode is indistinguishable +from a real cancel regression, which is the expensive part: it trains you to +re-run rather than to read. + +Requires `--dist=loadgroup` (set in pyproject.toml). With plain `--dist=load` the +marker is silently ignored -- see `test_conftest_serial_process.py`, which fails +if that ever regresses. + +This is not a substitute for fixing genuinely flaky assertions. Where a test +asserted something it had no business asserting -- a +/-1 second window on a +child's output, say -- that assertion was removed rather than protected by this. +""" + + +@pytest.fixture(autouse=True) +def _quiesce_after_process_test(request: pytest.FixtureRequest) -> Generator[None, None, None]: + """Between serial-process tests, wait for the timers and threads the previous + one created to actually go away. + + A `ScriptRunnerBase` leaves a `threading.Timer` running for the whole of an + unexpired timeout or cancel grace period, and a `ThreadPoolExecutor` worker + behind it. A test that finishes early -- because it cancelled its child -- can + therefore hand a live 30-second timer and a busy thread to whatever runs next. + Serialising the tests only helps if they also stop overlapping in that way. + + Cancels stray timers, then waits briefly for the thread count to settle. Does + not assert: a leftover thread is not necessarily this test's fault, and turning + that into a failure here would report it against the wrong test. It is logged + so it is visible when it matters. + """ + marker = request.node.get_closest_marker("xdist_group") + if marker is None or SERIAL_PROCESS_GROUP not in marker.args: + yield + return + + import threading + + before = threading.active_count() + yield + + for thread in threading.enumerate(): + if isinstance(thread, threading.Timer) and thread.is_alive(): + thread.cancel() + + # Deliberately short. Some tests legitimately leave a daemon stdout-reader + # thread behind that never exits (documented in LoggingSubprocess), so a + # generous budget here is spent in full on every one of them -- measured at 5s + # of pure teardown for a single test. One second is enough for a cancelled + # timer and a pool worker to wind down, which is what this is for. + deadline = time.monotonic() + 1.0 + while threading.active_count() > before and time.monotonic() < deadline: + time.sleep(0.02) + if threading.active_count() > before: + print( + f"\n[quiesce] {request.node.name} left " + f"{threading.active_count() - before} extra thread(s) running" + ) + + +def nonexistent_group_name() -> str: + """A group name that cannot resolve on any host. + + Randomized rather than hardcoded so that it cannot collide with a real group + on a developer's machine or in a test container. + """ + return f"openjd-no-such-group-{uuid.uuid4().hex}" + + +def resolvable_member_groups() -> list[tuple[int, str]]: + """The (gid, name) of every group this process is a member of and that has a + name in the group database. + + Discovered at runtime on purpose: this suite runs on macOS and Linux hosts + (and inside test containers) whose group tables have nothing in common, so no + group name can be hardcoded. The KeyError branch matters: a process can hold a + gid that the group database has no entry for, and asking for its name raises. + """ + groups: list[tuple[int, str]] = [] + for gid in sorted(set(os.getgroups()) | {os.getegid()}): # type: ignore + try: + groups.append((gid, grp.getgrgid(gid).gr_name)) # type: ignore + except KeyError: + # A gid the process holds that has no entry in the group database. + continue + return groups + + def create_unique_logger_name(prefix: str = "", seed: Optional[str] = None) -> str: """Create a unique logger name using a hash to avoid collisions. diff --git a/test/openjd/sessions_v0/test_action_filter_hardening.py b/test/openjd/sessions_v0/test_action_filter_hardening.py new file mode 100644 index 00000000..6446dda8 --- /dev/null +++ b/test/openjd/sessions_v0/test_action_filter_hardening.py @@ -0,0 +1,365 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Hardening of the action-output filter: log redaction, and containment of +consumer callbacks. + +``ActionMonitoringFilter`` runs on the thread forwarding a subprocess's stdout, +so anything that escapes it unwinds ``LoggingSubprocess.run()`` and costs us the +output stream and process ownership. It is also the control that keeps secrets +out of the log. +""" + +import logging +import sys +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CommandString as CommandString_2023_09, + StepActions as StepActions_2023_09, + StepScript as StepScript_2023_09, +) + +from openjd.sessions import ActionState, Session, SessionState +from openjd.sessions._action_filter import ( + ActionMessageKind, + ActionMonitoringFilter, + envvar_set_matcher_json, + envvar_set_matcher_str, + envvar_unset_matcher, +) + + +def _make_record( + msg: str, args: Any = None, session_id: str = "foo", level: int = logging.INFO +) -> logging.LogRecord: + """A LogRecord shaped the way the session logger produces them.""" + record = logging.LogRecord("test", level, "path", 1, msg, args, None) + record.session_id = session_id # type: ignore[attr-defined] + return record + + +class _UnrenderableError(Exception): + """An exception whose rendering raises -- the shape that defeated the R5-2 + containment, which interpolated the exception into an f-string.""" + + def __str__(self) -> str: + raise RuntimeError("__str__ is hostile") + + def __repr__(self) -> str: + raise RuntimeError("__repr__ is hostile too") + + +class TestRedactionDoesNotLeakViaRecordArgs: + """R5-1: `record.args` must be empty by the time the filter returns. + + The redaction logic only ever inspects `record.msg`. A downstream handler + calls `record.getMessage()`, which re-runs `msg % args` -- so any path that + leaves `args` populated re-interpolates the *un-scanned* original into the + emitted line. + """ + + def _filter_with_secret(self, secret: str = "SUPERSECRET") -> ActionMonitoringFilter: + f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) + f._redacted_values.add(secret) + return f + + def test_secret_in_args_is_redacted_when_formatting_succeeds(self) -> None: + # GIVEN: a secret carried only in args, with a msg that matches nothing + f = self._filter_with_secret() + record = _make_record("value is %s", ("SUPERSECRET",)) + + # WHEN + f.filter(record) + + # THEN: the emitted line -- which is what a handler actually renders -- + # carries no secret, and args cannot reintroduce one. + assert "SUPERSECRET" not in record.getMessage() + assert not record.args + + def test_secret_in_args_is_redacted_when_formatting_fails(self) -> None: + """The load-bearing case. `%d` against a str raises, so the old code + skipped clearing args and the handler re-interpolated the secret.""" + # GIVEN: a record whose own %-formatting is broken + f = self._filter_with_secret() + record = _make_record("value is %d", ("SUPERSECRET",)) + + # WHEN + f.filter(record) + + # THEN: args are cleared, so getMessage() cannot resurrect the secret, + # and the secret does not survive anywhere on the record. + assert not record.args + assert "SUPERSECRET" not in record.getMessage() + assert "SUPERSECRET" not in str(record.msg) + + def test_record_stays_renderable_when_formatting_fails(self) -> None: + """Folding args in must not leave a record that raises in the handler.""" + # GIVEN + f = self._filter_with_secret() + record = _make_record("value is %d", ("SUPERSECRET",)) + + # WHEN + f.filter(record) + + # THEN: getMessage() does not raise (it would if msg still held %d and + # args were still a str tuple), and the redaction marker is present. + assert "*" * 8 in record.getMessage() + + def test_whole_line_redaction_clears_args(self) -> None: + # GIVEN: a message that matches a whole redacted line + f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) + f._redacted_lines.add("secret-line") + record = _make_record("secret-line", None) + + # WHEN + f.filter(record) + + # THEN + assert record.getMessage() == "*" * 8 + assert not record.args + + def test_args_are_untouched_when_no_redactions_registered(self) -> None: + """Lazy %-formatting must keep working for every ordinary log line.""" + # GIVEN: no registered secrets + f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) + record = _make_record("value is %s", ("plain",)) + + # WHEN + f.filter(record) + + # THEN: the filter did not fold args in, so the handler still formats. + assert record.args == ("plain",) + assert record.getMessage() == "value is plain" + + +class TestFilterContainsConsumerCallbackFailures: + """R5-2: this filter runs on the thread forwarding the subprocess's stdout. + + An exception escaping `filter()` unwinds `LoggingSubprocess.run()` before the + child is waited on: the pump thread dies, the rest of the output is dropped, + and the process is left unreaped. + """ + + @pytest.mark.parametrize( + "exc", + [RuntimeError("boom"), KeyError("boom"), TypeError("boom"), AttributeError("boom")], + ids=["RuntimeError", "KeyError", "TypeError", "AttributeError"], + ) + def test_handler_callback_exception_does_not_escape(self, exc: Exception) -> None: + # GIVEN: a consumer callback that raises a non-ValueError + def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: + raise exc + + f = ActionMonitoringFilter(session_id="foo", callback=callback) + record = _make_record("openjd_progress: 50.0") + + # WHEN / THEN: filter() returns normally... + assert f.filter(record) is True + # ...and keeps the record, with the failure visible in the action's output. + assert "boom" in record.getMessage() + + def test_malformed_env_callback_exception_does_not_escape(self) -> None: + """The one callback invocation in filter() not routed through `handler`.""" + + # GIVEN + def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: + raise RuntimeError("boom") + + f = ActionMonitoringFilter(session_id="foo", callback=callback) + # A near-miss env command: space before the colon. + record = _make_record("openjd_env : FOO=bar") + + # WHEN / THEN + assert f.filter(record) is True + + def test_valueerror_still_annotates_the_record(self) -> None: + """The pre-existing ValueError contract must be unchanged.""" + # GIVEN: progress outside the legal range raises ValueError in the handler + f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) + record = _make_record("openjd_progress: 500.0") + + # WHEN / THEN + assert f.filter(record) is True + assert "ERROR" in record.getMessage() + + def test_redaction_failure_fails_closed(self) -> None: + """If the redaction control itself breaks, emit nothing rather than an + unscanned line -- and do not let it reach the pump thread.""" + # GIVEN + f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) + record = _make_record("carries a secret") + + # WHEN + with patch.object( + ActionMonitoringFilter, + "apply_message_redaction", + side_effect=RuntimeError("redaction is broken"), + ): + result = f.filter(record) + + # THEN + assert result is True + assert record.getMessage() == "*" * 8 + + def test_a_live_child_is_still_reaped_when_the_callback_raises(self) -> None: + """End to end: the reason R5-2 matters. A progress update from a live + child must not cost us the process.""" + # GIVEN: a consumer that raises on the first progress update + state: dict[str, Any] = {"raised": False} + + def callback(session_id: str, status: Any) -> None: + if status.progress is not None and not state["raised"]: + state["raised"] = True + raise RuntimeError("consumer blew up on a progress update") + + script = StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09(sys.executable), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09( + "print('openjd_progress: 50.0', flush=True)\n" + "print('done', flush=True)\n" + ), + ], + ) + ) + ) + + # WHEN + with Session(session_id="r5-2-e2e", job_parameter_values={}, callback=callback) as session: + session.run_task(step_script=script, task_parameter_values={}) + deadline = 60.0 + import time + + start = time.monotonic() + while session.state == SessionState.RUNNING and time.monotonic() - start < deadline: + time.sleep(0.05) + + # THEN: the consumer did raise, the action still reached a terminal + # state, and the subprocess was waited on -- an exit code proves the + # `wait()` in LoggingSubprocess.run() was reached rather than skipped. + assert state["raised"] is True + assert session.state != SessionState.RUNNING + assert session.action_status is not None + assert session.action_status.exit_code == 0 + assert session.action_status.state == ActionState.SUCCESS + + +class TestEnvVarNameAnchoring: + """`$` also matches immediately before a trailing newline, so an + `$`-anchored NAME pattern accepted "FOO\\n" -- a name no OS can hold. + + The VALUE half stays deliberately permissive: a multi-line value delivered + through the JSON form is supported, tested behaviour. + """ + + @pytest.mark.parametrize("name", ["FOO\n", "FOO\r", "FOO\r\n"]) + def test_unset_rejects_a_trailing_newline_in_the_name(self, name: str) -> None: + assert envvar_unset_matcher.match(name) is None + + def test_unset_still_accepts_a_legal_name(self) -> None: + assert envvar_unset_matcher.match("FOO_BAR9") is not None + + @pytest.mark.parametrize("payload", ["FOO=bar\n", "FOO\n=bar"]) + def test_set_rejects_a_trailing_newline(self, payload: str) -> None: + assert envvar_set_matcher_str.match(payload) is None + + def test_multiline_value_via_json_is_still_supported(self) -> None: + """Guards the intended feature against over-correction.""" + # GIVEN / WHEN + raw = '"FOO=BAR\\nBAZ"' + # THEN: the raw (escaped) form still validates... + assert envvar_set_matcher_json.match(raw) is not None + # ...and the decoded multi-line value still reaches the callback. + callback = MagicMock() + f = ActionMonitoringFilter(session_id="foo", callback=callback) + f.filter(_make_record('openjd_env: "FOO=BAR\\nBAZ"')) + env_calls = [ + c + for c in callback.call_args_list + if c[0][0] == ActionMessageKind.ENV and isinstance(c[0][1], dict) + ] + assert len(env_calls) == 1 + assert env_calls[0][0][1] == {"name": "FOO", "value": "BAR\nBAZ"} + + @pytest.mark.parametrize( + "msg", + [ + 'openjd_env: "FOO\\nBAR=baz"', + 'openjd_env: "FOO\\u000aBAR=baz"', + 'openjd_env: "FOO\\u0000BAR=baz"', + ], + ) + def test_a_separator_cannot_reach_a_decoded_name(self, msg: str) -> None: + """A name carrying a separator must be rejected outright, not passed on. + + Rewritten after an audit: the previous version looped over the recorded + calls asserting a property of any dict it found, and since these inputs + produce no ENV dict at all, its assertion body never executed. It passed + against a deliberately broken implementation. Assert the rejection + directly instead. + """ + # GIVEN + callback = MagicMock() + f = ActionMonitoringFilter(session_id="foo", callback=callback) + + # WHEN + f.filter(_make_record(msg)) + + # THEN: no environment variable was defined, and the failure was reported. + env_defs = [ + c + for c in callback.call_args_list + if c[0][0] == ActionMessageKind.ENV and isinstance(c[0][1], dict) + ] + assert env_defs == [] + assert callback.call_args_list, "the parse failure must be reported to the consumer" + assert callback.call_args_list[-1][0][2] is True # cancel-and-fail + + +class TestContainmentDoesNotReRaise: + def test_handler_path_contains_an_unrenderable_exception(self) -> None: + # GIVEN: a consumer callback raising an exception that cannot be rendered + def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: + raise _UnrenderableError() + + f = ActionMonitoringFilter(session_id="foo", callback=callback) + record = _make_record("openjd_progress: 50.0") + + # WHEN / THEN: filter() still returns rather than letting the exception + # reach the stdout pump thread. + assert f.filter(record) is True + assert "_UnrenderableError" in record.getMessage() + + def test_malformed_env_path_contains_an_unrenderable_exception(self) -> None: + # GIVEN + def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: + raise _UnrenderableError() + + f = ActionMonitoringFilter(session_id="foo", callback=callback) + + # WHEN / THEN + assert f.filter(_make_record("openjd_env : FOO=bar")) is True + + def test_renderable_exceptions_still_report_their_message(self) -> None: + """The defensive rendering must not degrade the ordinary case.""" + + # GIVEN + def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: + raise RuntimeError("a perfectly ordinary boom") + + f = ActionMonitoringFilter(session_id="foo", callback=callback) + record = _make_record("openjd_progress: 50.0") + + # WHEN + f.filter(record) + + # THEN + assert "a perfectly ordinary boom" in record.getMessage() diff --git a/test/openjd/sessions_v0/test_concurrency_fixes.py b/test/openjd/sessions_v0/test_concurrency_fixes.py new file mode 100644 index 00000000..0025c27f --- /dev/null +++ b/test/openjd/sessions_v0/test_concurrency_fixes.py @@ -0,0 +1,359 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for concurrency fixes F1-F8 from PR #333 review findings. + +These tests verify the defensive behaviors added to close race windows. +True concurrency races are inherently non-deterministic, so these tests +focus on verifying the code paths and edge-case handling rather than +timing-dependent race reproduction. +""" + +import sys +import threading +from datetime import timedelta +from logging.handlers import QueueHandler +from pathlib import Path +from queue import SimpleQueue +from unittest.mock import MagicMock, patch + +import pytest + +from openjd.model import SymbolTable +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CommandString as CommandString_2023_09, + EnvironmentActions as EnvironmentActions_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, +) +from openjd.sessions import ActionState +from openjd.sessions._runner_env_script import EnvironmentScriptRunner +from openjd.sessions._subprocess import LoggingSubprocess + +from .conftest import build_logger +from .conftest import serial_process + + +@pytest.mark.usefixtures("message_queue", "queue_handler") +class TestF1EnvScriptCancelDuringSetup: + """F1: Cancel during environment script setup must be recorded as pending.""" + + def test_cancel_before_action_assignment_records_pending( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + python_exe: str, + ) -> None: + """Verify cancel() before _action is assigned records pending cancel.""" + # GIVEN: An EnvironmentScriptRunner with no _action yet + logger = build_logger(queue_handler) + script = EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09("print('test')")], + ) + ) + ) + symtab = SymbolTable(source={}) + + runner = EnvironmentScriptRunner( + logger=logger, + session_working_directory=tmp_path, + environment_script=script, + symtab=symtab, + session_files_directory=tmp_path, + ) + + # Verify _action is None (before enter() or exit() is called) + assert runner._action is None + + # WHEN: cancel() is called before _action is set + runner.cancel(time_limit=timedelta(seconds=5), mark_action_failed=True) + + # THEN: The cancel is recorded as pending (not silently dropped) + assert runner._pending_cancel == (timedelta(seconds=5), True) + + runner.shutdown() + + +@pytest.mark.usefixtures("message_queue", "queue_handler") +class TestF3MonotonicMergePendingCancels: + """F3: Duplicate pending cancels must merge monotonically.""" + + def test_pending_cancel_merge_takes_minimum_time_limit( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + python_exe: str, + ) -> None: + """Verify multiple pending cancels merge with min(time_limit).""" + logger = build_logger(queue_handler) + script = EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09("print('test')")], + ) + ) + ) + symtab = SymbolTable(source={}) + + runner = EnvironmentScriptRunner( + logger=logger, + session_working_directory=tmp_path, + environment_script=script, + symtab=symtab, + session_files_directory=tmp_path, + ) + + # WHEN: First cancel with 10 second limit + runner.cancel(time_limit=timedelta(seconds=10), mark_action_failed=False) + assert runner._pending_cancel == (timedelta(seconds=10), False) + + # AND: Second cancel with 5 second limit (tighter) + runner.cancel(time_limit=timedelta(seconds=5), mark_action_failed=False) + + # THEN: Merged result takes minimum time limit + assert runner._pending_cancel == (timedelta(seconds=5), False) + + runner.shutdown() + + def test_pending_cancel_merge_or_mark_action_failed( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + python_exe: str, + ) -> None: + """Verify multiple pending cancels OR the mark_action_failed flags.""" + logger = build_logger(queue_handler) + script = EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09("print('test')")], + ) + ) + ) + symtab = SymbolTable(source={}) + + runner = EnvironmentScriptRunner( + logger=logger, + session_working_directory=tmp_path, + environment_script=script, + symtab=symtab, + session_files_directory=tmp_path, + ) + + # WHEN: First cancel without mark_action_failed + runner.cancel(time_limit=timedelta(seconds=10), mark_action_failed=False) + assert runner._pending_cancel == (timedelta(seconds=10), False) + + # AND: Second cancel with mark_action_failed=True + runner.cancel(time_limit=timedelta(seconds=15), mark_action_failed=True) + + # THEN: Merged result ORs the failed flags (once failed, always failed) + # and takes the minimum time limit + assert runner._pending_cancel == (timedelta(seconds=10), True) + + runner.shutdown() + + def test_pending_cancel_merge_none_beats_unlimited( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + python_exe: str, + ) -> None: + """Verify that a defined limit beats None (unlimited).""" + logger = build_logger(queue_handler) + script = EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09("print('test')")], + ) + ) + ) + symtab = SymbolTable(source={}) + + runner = EnvironmentScriptRunner( + logger=logger, + session_working_directory=tmp_path, + environment_script=script, + symtab=symtab, + session_files_directory=tmp_path, + ) + + # WHEN: First cancel with unlimited time (None) + runner.cancel(time_limit=None, mark_action_failed=False) + assert runner._pending_cancel == (None, False) + + # AND: Second cancel with defined limit + runner.cancel(time_limit=timedelta(seconds=5), mark_action_failed=False) + + # THEN: Defined limit wins over unlimited + assert runner._pending_cancel == (timedelta(seconds=5), False) + + runner.shutdown() + + +@pytest.mark.usefixtures("message_queue", "queue_handler") +class TestF7SelfJoinDetection: + """F7: shutdown() must not deadlock when called from worker thread.""" + + def test_shutdown_from_worker_thread_uses_wait_false( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + python_exe: str, + ) -> None: + """Verify shutdown() detects self-join and uses wait=False.""" + logger = build_logger(queue_handler) + script = EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09("print('test')")], + ) + ) + ) + symtab = SymbolTable(source={}) + + runner = EnvironmentScriptRunner( + logger=logger, + session_working_directory=tmp_path, + environment_script=script, + symtab=symtab, + session_files_directory=tmp_path, + ) + + # Simulate calling shutdown from the pool's worker thread + # by adding current thread to _threads set + threads = runner._pool._threads + if isinstance(threads, set): + threads.add(threading.current_thread()) + + # WHEN: shutdown() is called (would deadlock if wait=True) + with patch.object(runner._pool, "shutdown") as mock_shutdown: + runner.shutdown() + + # THEN: shutdown was called with wait=False + mock_shutdown.assert_called_once_with(wait=False) + + +@pytest.mark.usefixtures("message_queue", "queue_handler") +class TestF8ObserverExceptionHandling: + """F8: Observer callback exceptions must not discard live children.""" + + def test_callback_exception_is_caught_and_logged( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + python_exe: str, + ) -> None: + """Verify callback exceptions don't propagate from _on_process_exit.""" + logger = build_logger(queue_handler) + + # Create a callback that raises an exception + def bad_callback(state: ActionState) -> None: + raise RuntimeError("Observer exploded!") + + script = EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09("print('test')")], + ) + ) + ) + symtab = SymbolTable(source={}) + + runner = EnvironmentScriptRunner( + logger=logger, + session_working_directory=tmp_path, + environment_script=script, + symtab=symtab, + session_files_directory=tmp_path, + callback=bad_callback, + ) + + # Simulate the state after a process has run + runner._process = MagicMock() + runner._process.is_running = False + mock_future = MagicMock() + mock_future.exception.return_value = None + runner._run_future = mock_future + + # WHEN: _on_process_exit is called (with bad callback) + # THEN: No exception should propagate + runner._on_process_exit(mock_future) + + # The runner should still be in a consistent state + runner.shutdown() + + +@pytest.mark.usefixtures("message_queue", "queue_handler") +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX-specific signal handling") +@serial_process +class TestReview22F4DoubleLoadFix: + """Review22-F4: notify/terminate must bind _process once and pass to helpers.""" + + def test_notify_binds_process_once( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + """Verify notify() binds _process once and passes to helper (R4-G8 fix).""" + logger = build_logger(queue_handler) + + subprocess = LoggingSubprocess( + logger=logger, + args=["true"], + working_dir=str(tmp_path), + ) + + # Set up a mock process + mock_process = MagicMock() + mock_process.poll.return_value = None # Process is running + subprocess._process = mock_process + + # WHEN: notify() is called + with patch.object(subprocess, "_posix_signal_subprocess") as mock_signal: + subprocess.notify() + + # THEN: Signal helper was called with the bound process (R4-G8) + mock_signal.assert_called_once_with(mock_process, signal_name="term") + + def test_terminate_binds_process_once( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + """Verify terminate() binds _process once and passes to helper (R4-G8 fix).""" + logger = build_logger(queue_handler) + + subprocess = LoggingSubprocess( + logger=logger, + args=["true"], + working_dir=str(tmp_path), + ) + + # Set up a mock process + mock_process = MagicMock() + mock_process.poll.return_value = None # Process is running + subprocess._process = mock_process + + # WHEN: terminate() is called + with patch.object(subprocess, "_posix_signal_subprocess") as mock_signal: + subprocess.terminate() + + # THEN: Signal helper was called with the bound process (R4-G8) + mock_signal.assert_called_once_with(mock_process, signal_name="kill") diff --git a/test/openjd/sessions_v0/test_conftest_serial_process.py b/test/openjd/sessions_v0/test_conftest_serial_process.py new file mode 100644 index 00000000..8a539e5a --- /dev/null +++ b/test/openjd/sessions_v0/test_conftest_serial_process.py @@ -0,0 +1,71 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""The `serial_process` mechanism must keep working, silently-failing being its +whole risk. + +`@pytest.mark.xdist_group` is only honoured under `--dist=loadgroup`. Under the +default `--dist=load` it is accepted, ignored, and reported nowhere -- so the +process-heavy tests would quietly go back to competing with eleven siblings and +the cancel/terminate family would start failing together again on loaded hosts, +looking exactly like a product regression. + +These tests are cheap and exist so that a change to `addopts` fails here, next to +an explanation, rather than as fourteen mystery failures somewhere else. +""" + +from __future__ import annotations + +import pytest + +from .conftest import SERIAL_PROCESS_GROUP, serial_process + + +def test_loadgroup_distribution_is_configured(pytestconfig: pytest.Config) -> None: + """Pins the `--dist=loadgroup` setting the marker depends on. + + Read from `addopts` rather than from `getoption("dist")`: inside an xdist + worker the resolved `dist` option is `"no"`, because a worker runs its own + share serially and only the controller distributes. Asserting on the resolved + option therefore fails on the workers and passes nowhere useful -- which is + exactly what the first version of this test did. + """ + # GIVEN / WHEN + addopts = " ".join(pytestconfig.getini("addopts")) + + # THEN + assert "--dist=loadgroup" in addopts, ( + "xdist_group markers are silently ignored unless --dist=loadgroup; the " + "serial_process tests would go back to running in parallel with each other" + ) + + +def test_serial_process_is_an_xdist_group_marker() -> None: + """The mark must be the one xdist looks for, with the expected group name. + + A rename or a typo would leave a decorator that reads as if it does something + and does nothing at all. + """ + # GIVEN / WHEN + mark = serial_process.mark + + # THEN + assert mark.name == "xdist_group" + assert mark.args == (SERIAL_PROCESS_GROUP,) + + +@serial_process +class TestMarkerReachesTheTest: + def test_the_marker_is_visible_on_a_decorated_test( + self, request: pytest.FixtureRequest + ) -> None: + """A class-level mark must actually reach the test item. + + This is what the quiesce fixture in conftest keys off, so if the mark stops + propagating the cleanup silently stops running too. + """ + # GIVEN / WHEN + marker = request.node.get_closest_marker("xdist_group") + + # THEN + assert marker is not None + assert SERIAL_PROCESS_GROUP in marker.args diff --git a/test/openjd/sessions_v0/test_embedded_files.py b/test/openjd/sessions_v0/test_embedded_files.py index 09b671b8..8cbf373d 100644 --- a/test/openjd/sessions_v0/test_embedded_files.py +++ b/test/openjd/sessions_v0/test_embedded_files.py @@ -1,5 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +import errno import os import stat import uuid @@ -26,6 +27,7 @@ EmbeddedFiles, EmbeddedFilesScope, _validate_embedded_filename, + chown_group, write_file_for_user, ) from openjd.sessions._session_user import PosixSessionUser, WindowsSessionUser @@ -33,6 +35,8 @@ from .conftest import ( has_posix_target_user, has_windows_user, + nonexistent_group_name, + resolvable_member_groups, WIN_SET_TEST_ENV_VARS_MESSAGE, POSIX_SET_TARGET_USER_ENV_VARS_MESSAGE, ) @@ -892,6 +896,48 @@ class Datum: result_contents = file.read() assert result_contents == expected_file_data, "File contents are as expected" + @pytest.mark.skipif(not is_posix(), reason="group ownership is posix-specific") + def test_unresolvable_group_fails_as_runtimeerror(self, tmp_path: Path) -> None: + """Pins: a group name that does not resolve must fail materialize() + through its existing handler, as RuntimeError. + + shutil.chown raises LookupError for an unknown group, and LookupError + is neither OSError nor ValueError -- the two classes + write_file_contents() catches -- so before chown_group() it escaped + this handler (and every other one in the chain) and surfaced as a + bare LookupError out of the public Session API. PosixSessionUser + does not validate its group, so a caller only has to pass a group + that does not exist. + + pytest.raises(RuntimeError) does not catch LookupError, so if the + translation is removed this test errors out with the escaping + LookupError -- which is exactly the defect. + """ + + # GIVEN + # Only `group` matters on this path; the user is never resolved by it. + user = PosixSessionUser(user="nobody", group=nonexistent_group_name()) + test_obj = EmbeddedFiles( + logger=MagicMock(), + scope=EmbeddedFilesScope.ENV, + session_files_directory=tmp_path, + user=user, + ) + given_file = EmbeddedFileText_2023_09( + name="Foo", + type=EmbeddedFileTypes_2023_09.TEXT, + data=DataString_2023_09("some data"), + ) + + # WHEN + with pytest.raises(RuntimeError) as excinfo: + test_obj.materialize([given_file], SymbolTable()) + + # THEN + # The failure is the group ownership change, not something incidental: + # the offending group name is carried through to the message. + assert user.group in str(excinfo.value) + class TestValidateEmbeddedFilename: """Unit tests for the _validate_embedded_filename() basename guard. @@ -947,3 +993,95 @@ def test_rejects_windows_non_basenames(self, filename: str) -> None: # WHEN / THEN with pytest.raises(ValueError): _validate_embedded_filename(filename) + + +@pytest.mark.skipif(not is_posix(), reason="shutil.chown's group handling is posix-only") +class TestChownGroup: + """Unit tests for chown_group(), the shutil.chown wrapper. + + Defect pinned: shutil.chown raises LookupError when the group name does not + resolve. LookupError is neither OSError nor ValueError, and every handler + around file materialization or session-directory setup in this package + catches some combination of OSError/ValueError/RuntimeError -- so an + unresolvable group escaped all of them and reached the caller of the public + Session API as a bare LookupError. chown_group() translates it at the call + site instead, because a group that cannot be resolved *is* a failure to + change ownership and callers already treat that as OSError. + """ + + def test_unresolvable_group_raises_oserror(self, tmp_path: Path) -> None: + """The translation itself: OSError out, not LookupError. + + pytest.raises(OSError) cannot catch a LookupError, so removing the + translation makes this test error out with the escaping LookupError. It + also fails if the wrapper raises some other class (e.g. ValueError) or + swallows the failure entirely. + """ + # GIVEN + filename = tmp_path / "file.txt" + filename.write_text("some data") + group = nonexistent_group_name() + + # WHEN + with pytest.raises(OSError) as excinfo: + chown_group(filename, group) + + # THEN + # Both the path and the group are named, so an operator can tell which + # file and which group configuration failed. + assert str(filename) in str(excinfo.value) + assert group in str(excinfo.value) + # AND the original LookupError is preserved as the cause, which proves + # this OSError is the translated group lookup and not an unrelated + # filesystem error. + assert isinstance(excinfo.value.__cause__, LookupError) + + def test_resolvable_group_succeeds(self, tmp_path: Path) -> None: + """The wrapper must not break the success path. + + The gid assertion is what makes this non-vacuous: it only holds if the + chown actually happened. A group other than the file's current group is + preferred so that the assertion cannot pass by accident. + """ + # GIVEN + candidates = resolvable_member_groups() + if not candidates: + pytest.skip("this process is not a member of any group that has a name") + filename = tmp_path / "file.txt" + filename.write_text("some data") + current_gid = filename.stat().st_gid + gid, group = next( + ((gid, name) for gid, name in candidates if gid != current_gid), candidates[0] + ) + + # WHEN + chown_group(filename, group) + + # THEN + assert filename.stat().st_gid == gid + + def test_real_oserror_is_passed_through_unchanged(self, tmp_path: Path) -> None: + """A genuine failure from shutil.chown must reach the caller as-is. + + The wrapper only exists to reclassify LookupError; if it broadened to + catch more than that, a real errno would be flattened into a generic + error and the handlers upstream would report the wrong cause. Using a + resolvable group with a missing path gets shutil.chown past its group + lookup and into os.chown, so the ENOENT here comes from the real syscall + rather than from a mock. + """ + # GIVEN + candidates = resolvable_member_groups() + if not candidates: + pytest.skip("this process is not a member of any group that has a name") + _gid, group = candidates[0] + missing = tmp_path / "no-such-file.txt" + + # WHEN + with pytest.raises(FileNotFoundError) as excinfo: + chown_group(missing, group) + + # THEN + # Same class, same errno, and no re-raise chain: it is the original error. + assert excinfo.value.errno == errno.ENOENT + assert excinfo.value.__cause__ is None diff --git a/test/openjd/sessions_v0/test_generated_shell_script.py b/test/openjd/sessions_v0/test_generated_shell_script.py new file mode 100644 index 00000000..fe4ec725 --- /dev/null +++ b/test/openjd/sessions_v0/test_generated_shell_script.py @@ -0,0 +1,179 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Quoting in the shell script generated for a POSIX action. + +The script is ``exec``’d by ``/bin/sh``, so a single unquoted metacharacter +anywhere in it is arbitrary code execution as the session user. +""" + +import sys +from datetime import timedelta +from logging.handlers import QueueHandler +from pathlib import Path +from queue import SimpleQueue +from typing import Optional +from unittest.mock import MagicMock + +import pytest + + +from openjd.sessions._os_checker import is_posix +from openjd.sessions._runner_base import POSIX_SHELL_NAME_RE, ScriptRunnerBase +from openjd.sessions._subprocess import LoggingSubprocess + +from .conftest import build_logger + + +class _Runner(ScriptRunnerBase): + """Minimal concrete runner: only _generate_command_shell_script is exercised.""" + + def cancel( + self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False + ) -> None: + pass + + +@pytest.mark.skipif(not is_posix(), reason="the generated shell script is POSIX-only") +class TestGeneratedShellScriptQuoting: + """The generated script is `exec`'d by /bin/sh, so a single unquoted + metacharacter anywhere in it is code execution as the session user.""" + + def _script( + self, + tmp_path: Path, + *, + os_env_vars: Optional[dict[str, Optional[str]]] = None, + startup_directory: Optional[Path] = None, + ) -> str: + runner = _Runner( + logger=MagicMock(), + session_working_directory=tmp_path, + os_env_vars=os_env_vars, + startup_directory=startup_directory, + ) + try: + return runner._generate_command_shell_script(["/bin/echo", "hi"]) + finally: + runner.shutdown() + + def test_single_quote_in_the_startup_directory_cannot_break_out(self, tmp_path: Path) -> None: + # GIVEN: a startup directory whose name closes the old hand-written quote + hostile = Path("/tmp/x'; touch /tmp/openjd_r5_pwned; '") + + # WHEN + script = self._script(tmp_path, startup_directory=hostile) + + # THEN: the injected command is inside a quoted word, not a command. + cd_line = next(line for line in script.splitlines() if line.startswith("cd ")) + assert "touch /tmp/openjd_r5_pwned" in cd_line # the path is preserved verbatim... + assert not cd_line.startswith("cd '/tmp/x'; ") # ...but no longer executes. + # Prove it to the shell itself rather than by eyeballing the quoting. + import shlex + + assert shlex.split(cd_line) == ["cd", str(hostile)] + + def test_ordinary_startup_directory_with_spaces_still_works(self, tmp_path: Path) -> None: + # GIVEN + spaced = tmp_path / "a dir with spaces" + + # WHEN + script = self._script(tmp_path, startup_directory=spaced) + + # THEN + import shlex + + cd_line = next(line for line in script.splitlines() if line.startswith("cd ")) + assert shlex.split(cd_line) == ["cd", str(spaced)] + + @pytest.mark.parametrize( + "hostile_name", + [ + "BAD;touch /tmp/openjd_r5_pwned;X", + "BAD$(touch /tmp/openjd_r5_pwned)", + "BAD`touch /tmp/openjd_r5_pwned`", + "BAD\ntouch /tmp/openjd_r5_pwned", + "BAD&&touch /tmp/openjd_r5_pwned", + ], + ) + def test_hostile_env_var_names_are_not_emitted(self, tmp_path: Path, hostile_name: str) -> None: + # GIVEN / WHEN + script = self._script(tmp_path, os_env_vars={hostile_name: "v", "GOOD": "ok"}) + + # THEN: the hostile name produced no line at all, and the legal one did. + assert "openjd_r5_pwned" not in script + assert "export GOOD=ok" in script + + def test_names_that_are_legal_elsewhere_but_not_in_sh_are_skipped(self, tmp_path: Path) -> None: + """`ProgramFiles(x86)` is a real Windows variable name and an outright + /bin/sh syntax error -- previously it broke the whole action.""" + # GIVEN / WHEN + script = self._script(tmp_path, os_env_vars={"ProgramFiles(x86)": "C:\\PF"}) + + # THEN + assert "ProgramFiles(x86)" not in script + + def test_generated_script_is_valid_sh_even_with_hostile_input(self, tmp_path: Path) -> None: + """The strongest assertion available: hand it to `sh -n`.""" + # GIVEN + script = self._script( + tmp_path, + os_env_vars={ + "BAD;X": "v", + "ProgramFiles(x86)": "v", + "GOOD": "a'b\"c$(d)", + "UNSET_ME": None, + }, + startup_directory=Path("/tmp/x'; touch /tmp/openjd_r5_pwned; '"), + ) + script_file = tmp_path / "candidate.sh" + script_file.write_text(script) + + # WHEN + from subprocess import run + + result = run(["/bin/sh", "-n", str(script_file)], capture_output=True, text=True) + + # THEN + assert result.returncode == 0, f"generated script is not valid sh: {result.stderr}" + + def test_unset_is_emitted_for_a_legal_name(self, tmp_path: Path) -> None: + # GIVEN / WHEN + script = self._script(tmp_path, os_env_vars={"GOES_AWAY": None}) + + # THEN + assert "unset GOES_AWAY" in script + + @pytest.mark.parametrize("name", ["FOO", "_foo", "F9", "_", "aB_9"]) + def test_posix_name_re_accepts_legal_identifiers(self, name: str) -> None: + assert POSIX_SHELL_NAME_RE.fullmatch(name) is not None + + @pytest.mark.parametrize("name", ["9FOO", "", "FOO BAR", "FOO=", "FOO\n", "FOO;", "é"]) + def test_posix_name_re_rejects_everything_else(self, name: str) -> None: + assert POSIX_SHELL_NAME_RE.fullmatch(name) is None + + def test_a_skipped_name_still_reaches_the_subprocess( + self, tmp_path: Path, message_queue: SimpleQueue, queue_handler: QueueHandler + ) -> None: + """Skipping the export line must not change what the child sees, because + Popen's `env=` does not go through a shell.""" + # GIVEN: a name that is illegal in sh but legal as an environment entry + logger = build_logger(queue_handler) + proc = LoggingSubprocess( + logger=logger, + args=[ + sys.executable, + "-c", + "import os;print('VAL=' + os.environ.get('ProgramFiles(x86)', 'MISSING'))", + ], + os_env_vars={"ProgramFiles(x86)": "present"}, + ) + + # WHEN + proc.run() + + # THEN + lines = [] + while not message_queue.empty(): + lines.append(message_queue.get().getMessage()) + assert "VAL=present" in lines + assert proc.exit_code == 0 diff --git a/test/openjd/sessions_v0/test_linux_sudo.py b/test/openjd/sessions_v0/test_linux_sudo.py new file mode 100644 index 00000000..56c3c948 --- /dev/null +++ b/test/openjd/sessions_v0/test_linux_sudo.py @@ -0,0 +1,650 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +import os +import shutil +import signal +import subprocess +import time +from logging.handlers import QueueHandler +from pathlib import Path +from queue import SimpleQueue +from subprocess import DEVNULL, PIPE, CompletedProcess, Popen +from typing import Callable, Generator, Optional +from unittest.mock import Mock, patch + +import pytest + +from openjd.sessions._linux._sudo import ( + FindSignalTargetError, + find_child_process_id_pgrep, + find_sudo_child_process_group_id, +) +from openjd.sessions._os_checker import is_posix + +from .conftest import build_logger, collect_queue_messages + +# pgrep's documented exit status for "no processes matched". Spelled out as a +# literal rather than imported from the module under test: a test that reused the +# module's own constant could not tell us whether that constant matches what the +# operating system actually returns. +PGREP_EXIT_NO_MATCH = 1 + +# The stand-in for sudo: a process that gains its child only after a delay, so a +# scan that starts immediately is guaranteed to see no children on its first look. +PARENT_SCRIPT = """ +import subprocess +import sys +import time + +python_exe, child_script, pgid_file, delay_seconds = sys.argv[1:5] +time.sleep(float(delay_seconds)) +# start_new_session, as sudo does, so the child lands in its own process group. +child = subprocess.Popen([python_exe, child_script, pgid_file], start_new_session=True) +child.wait() +""" + +# The stand-in for the workload. It reports its own process group id, so the test +# asserts against the group the kernel actually assigned rather than recomputing +# it the same way the code under test does. +CHILD_SCRIPT = """ +import os +import sys +import time + +pgid_file = sys.argv[1] +# Written to a temporary name and renamed, so a reader never sees a partial id. +tmp_file = pgid_file + ".tmp" +with open(tmp_file, "w") as f: + f.write(str(os.getpgid(0))) +os.replace(tmp_file, pgid_file) +time.sleep(15) +""" + + +# A process that moves into its own process group only after a delay, reproducing +# the window sudo leaves open: it forks first and creates the new process group +# second, so a scan can legitimately observe the child still sharing sudo's group. +# +# setsid() rather than setpgid(): the process is started without +# start_new_session, so it is not already a group leader and setsid() cannot fail +# with EPERM. It reports the group it ends up in, so the test asserts against what +# the kernel assigned rather than recomputing it. +LATE_PROCESS_GROUP_SCRIPT = """ +import os +import time + +pgid_file = %(pgid_file)s +time.sleep(%(delay_seconds)s) +os.setsid() +tmp_file = pgid_file + ".tmp" +with open(tmp_file, "w") as f: + f.write(str(os.getpgid(0))) +os.replace(tmp_file, pgid_file) +time.sleep(15) +""" + + +def pgrep_result(returncode: int, stdout: str = "") -> CompletedProcess: + """A stand-in for the CompletedProcess of a `pgrep -P ` run.""" + return CompletedProcess(args=["pgrep", "-P", "1234"], returncode=returncode, stdout=stdout) + + +@pytest.fixture +def sleeper(python_exe: str) -> Generator[Callable[..., Popen], None, None]: + """Factory for live, long-lived child processes, all killed and reaped at teardown. + + Real processes rather than fabricated pids because the code under test calls + os.getpgid() on whatever the scan returns: a made-up pid would either raise + ProcessLookupError or -- worse, if the number happened to be live -- report + the process group of something unrelated. + + Registration happens inside the factory, so a process cannot be leaked by a + failure between starting it and the caller's first statement. + """ + procs: list[Popen] = [] + + def start(*, own_process_group: bool = False, script: Optional[str] = None) -> Popen: + argv = [python_exe, "-c", script if script else "import time; time.sleep(30)"] + proc = Popen( + argv, + stdin=DEVNULL, + stdout=DEVNULL, + # A scripted child can die on startup, and without its stderr the test + # that depends on it fails with no explanation. The plain sleeper + # cannot fail that way, so it keeps the pipe-free form. + stderr=PIPE if script else DEVNULL, + # start_new_session is what sudo does: the child becomes a session and + # process-group leader, so its pgid is its own pid and is knowable + # without asking the code under test. + start_new_session=own_process_group, + ) + procs.append(proc) + return proc + + yield start + + for proc in procs: + # Popen.kill() is a no-op once the process has been reaped, so this is safe + # for the already-reaped process the short-circuit test needs. + proc.kill() + try: + # communicate() rather than wait(): it drains the stderr pipe as well + # as reaping, so a child that wrote to stderr cannot wedge teardown. + proc.communicate(timeout=10) + except ValueError: + # A test on its failure path already read and closed that pipe. + proc.wait(timeout=10) + + +def stderr_of(proc: Popen) -> bytes: + """The child's stderr, for the diagnostic on a failing assertion. + + Killed first: this is only reached when the child has already failed to do + what a test needed of it, and a still-sleeping child would otherwise hold + communicate() open for the rest of its life. + """ + proc.kill() + return proc.communicate(timeout=10)[1] or b"" + + +def poll_for_pgid(pgid_file: Path, timeout_seconds: float) -> Optional[int]: + """The process group id reported by the child, or None if it never reported. + + Polled rather than slept on, and a None return makes the caller's assertion + fail loudly instead of the test quietly passing on a missing file. The file + is always read at least once, so a zero timeout still answers. + """ + deadline = time.monotonic() + timeout_seconds + while True: + try: + return int(pgid_file.read_text()) + except (FileNotFoundError, ValueError): + if time.monotonic() >= deadline: + return None + time.sleep(0.01) + + +@pytest.mark.skipif(not is_posix(), reason="pgrep and process groups are posix-only") +class TestFindChildProcessIdPgrep: + """Tests for find_child_process_id_pgrep(), the non-Linux POSIX lookup of + sudo's child process. + + Defect pinned: this function raised FindSignalTargetError for *any* non-zero + pgrep exit. Exit 1 means "no processes matched", which is the expected answer + for most of the caller's retry window -- sudo has forked but the kernel has + not finished creating the workload yet. Because the caller's + `except FindSignalTargetError` sits outside its retry loop, that first empty + poll ended all retries, so on every non-Linux POSIX host a cross-user launch + recorded no process group and a later cancel had nothing to signal. + """ + + def test_no_matching_process_returns_none(self, python_exe: str) -> None: + """Exit 1 is an answer ("none yet"), not a failure. + + Run against a real, live process that has no children of its own, so it + exercises the real pgrep rather than an assumption about it. The probe + below asserts the operating system's half of the contract -- that this + situation really is exit 1 with no output -- and the `result is None` + assertion is what fails if exit 1 raises again. + """ + # GIVEN a live process with no children of its own + proc = Popen( + [python_exe, "-c", "import time; time.sleep(15)"], + stdin=DEVNULL, + stdout=DEVNULL, + stderr=DEVNULL, + ) + try: + probe = subprocess.run( + ["pgrep", "-P", str(proc.pid)], + stdin=DEVNULL, + capture_output=True, + text=True, + ) + assert ( + probe.returncode == PGREP_EXIT_NO_MATCH + ), f"pgrep did not report 'no match' as exit {PGREP_EXIT_NO_MATCH}: {probe}" + assert probe.stdout.strip() == "", f"pgrep unexpectedly found children: {probe.stdout}" + + # WHEN + result = find_child_process_id_pgrep(sudo_pid=proc.pid) + + # THEN + assert result is None + finally: + proc.kill() + proc.wait() + + @pytest.mark.parametrize( + "returncode,output", + [ + pytest.param(2, "pgrep: illegal option -- Q\n", id="usage-error"), + pytest.param(3, "pgrep: cannot open kernel memory\n", id="fatal-error"), + pytest.param(127, "sh: pgrep: command not found\n", id="not-found"), + ], + ) + def test_other_nonzero_exits_still_raise(self, returncode: int, output: str) -> None: + """The other half of the fix: only exit 1 is benign. + + Treating every non-zero exit as "no child yet" would silently turn a + broken or missing pgrep into "this sudo has no workload", which is the + same lost-signal-target outcome by a different route. + + Both halves of the message are pinned. The exit code alone names a + category of failure without naming the failure: this call merges stderr + into stdout, so the message is the only place pgrep's own explanation + survives, and the exception is caught and logged as a warning rather + than propagated, so there is no traceback to fall back on. + """ + # GIVEN + with patch( + "openjd.sessions._linux._sudo.run", return_value=pgrep_result(returncode, output) + ): + # WHEN + with pytest.raises(FindSignalTargetError) as excinfo: + find_child_process_id_pgrep(sudo_pid=1234) + + # THEN + message = str(excinfo.value) + assert f"exited {returncode}" in message + # repr'd, and so quoted: an operator can see leading/trailing whitespace + # and can tell an empty explanation from a missing one. + assert repr(output.strip()) in message + + def test_single_child_is_returned(self) -> None: + """Exit 0 with one pid: the pid is the answer.""" + # GIVEN + with patch("openjd.sessions._linux._sudo.run", return_value=pgrep_result(0, "4321\n")): + # WHEN + result = find_child_process_id_pgrep(sudo_pid=1234) + + # THEN + assert result == 4321 + + def test_multiple_children_raise(self) -> None: + """Exit 0 with several pids violates the assumption that sudo has exactly + one child, so it must not be resolved by guessing one of them.""" + # GIVEN + with patch( + "openjd.sessions._linux._sudo.run", return_value=pgrep_result(0, "4321\n4322\n") + ): + # WHEN + with pytest.raises(FindSignalTargetError) as excinfo: + find_child_process_id_pgrep(sudo_pid=1234) + + # THEN + assert "4321" in str(excinfo.value) and "4322" in str(excinfo.value) + + def test_no_output_returns_none(self) -> None: + """Exit 0 with no pids is the same "none yet" answer as exit 1, and must + stay distinguishable from it only in how it is reached.""" + # GIVEN + with patch("openjd.sessions._linux._sudo.run", return_value=pgrep_result(0, "")): + # WHEN + result = find_child_process_id_pgrep(sudo_pid=1234) + + # THEN + assert result is None + + +@pytest.mark.skipif(not is_posix(), reason="pgrep and process groups are posix-only") +class TestFindSudoChildProcessGroupId: + """Tests for find_sudo_child_process_group_id() over the pgrep lookup.""" + + def test_keeps_scanning_until_a_late_child_appears( + self, + tmp_path: Path, + python_exe: str, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + """The behaviour the fix exists for: an empty scan must not end the + retries. + + A parent that gains its child only after a delay reproduces the real + race -- sudo forks before the kernel finishes creating the workload. When + an empty pgrep raised FindSignalTargetError, the caller's handler (which + sits outside its retry loop) swallowed it, logged a warning and returned + None, so the process group was never recorded and a later cancel had + nothing to signal. Asserting the *correct* pgid, not merely "not None", + is what makes this a pin of the fix rather than of the constant. + + is_linux is forced False so that the pgrep path is the one exercised on + every POSIX host; on Linux the caller would otherwise take the procfs + path, which already returns None for this situation. + """ + # GIVEN + if shutil.which("pgrep") is None: + pytest.skip("pgrep is not installed on this host") + parent_script = tmp_path / "parent.py" + parent_script.write_text(PARENT_SCRIPT) + child_script = tmp_path / "child.py" + child_script.write_text(CHILD_SCRIPT) + pgid_file = tmp_path / "child_pgid.txt" + child_appears_after_seconds = 0.4 + logger = build_logger(queue_handler) + + parent = Popen( + [ + python_exe, + str(parent_script), + python_exe, + str(child_script), + str(pgid_file), + str(child_appears_after_seconds), + ], + stdin=DEVNULL, + stdout=DEVNULL, + stderr=DEVNULL, + ) + child_pgid: Optional[int] = None + try: + # WHEN + with patch("openjd.sessions._linux._sudo.is_linux", return_value=False): + # A window far wider than the delay, so a slow host cannot fail + # this spuriously. The scan returns as soon as it finds the + # child, and the defect returns None on the first poll, so the + # wide window costs nothing either way. + result = find_sudo_child_process_group_id( + logger=logger, + sudo_process=parent, + timeout_seconds=10.0, + ) + + # THEN + child_pgid = poll_for_pgid(pgid_file, timeout_seconds=5.0) + assert child_pgid is not None, ( + "the child never reported its process group, so this test could not have " + f"observed one (parent exit code: {parent.poll()})" + ) + assert result == child_pgid + # AND the scan did not report a failure along the way. + messages = collect_queue_messages(message_queue) + assert not any( + "Unable to determine signal target" in message for message in messages + ), messages + finally: + parent.kill() + parent.wait() + if child_pgid is None: + child_pgid = poll_for_pgid(pgid_file, timeout_seconds=0) + if child_pgid is not None: + try: + os.killpg(child_pgid, signal.SIGKILL) # type: ignore + except (ProcessLookupError, PermissionError): + # Best-effort teardown. The group is already gone + # (ProcessLookupError) or was never ours to signal + # (PermissionError); either way there is nothing left to clean + # up and failing here would mask the test's real result. + pass + + +@pytest.mark.skipif(not is_posix(), reason="pgrep and process groups are posix-only") +class TestFindSudoChildProcessGroupIdScanErrors: + """Tests that a single bad scan does not end the retries. + + Defect pinned: the `except FindSignalTargetError` that guards the scan used to + sit OUTSIDE the retry `while`, so the *first* bad scan aborted the whole + search -- the process group was never recorded, and a later cancel had + nothing to signal. Every condition the loop races is transient by + construction: the procfs scan sees more than one child while sudo's fork is + still settling, and a `pgrep` invocation can fail for reasons that do not + recur on the next poll. + + The scan is mocked here, unlike the end-to-end test above, because a *failing* + scan is not something a real host can be asked for on demand -- and both + branches (procfs and pgrep) have to be covered from one platform. + """ + + @pytest.mark.parametrize( + "on_linux,scan_function", + [ + pytest.param(True, "find_sudo_child_process_id_procfs", id="procfs"), + pytest.param(False, "find_child_process_id_pgrep", id="pgrep"), + ], + ) + def test_transient_scan_error_does_not_end_the_retries( + self, + on_linux: bool, + scan_function: str, + sleeper: Callable[..., Popen], + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + """One raising scan followed by a succeeding one still yields the pgid. + + Parametrized over both lookups because the two are selected by is_linux() + and the defect was in the shared caller, so a fix verified on one branch + says nothing about the other. + """ + # GIVEN + sudo_process = sleeper() + # The workload, in its own process group as sudo's child would be. + workload = sleeper(own_process_group=True) + logger = build_logger(queue_handler) + scan = Mock(side_effect=[FindSignalTargetError("transient"), workload.pid]) + + # WHEN + with ( + patch(f"openjd.sessions._linux._sudo.{scan_function}", scan), + patch("openjd.sessions._linux._sudo.is_linux", return_value=on_linux), + ): + result = find_sudo_child_process_group_id( + logger=logger, + sudo_process=sudo_process, + timeout_seconds=10.0, + ) + + # THEN + # start_new_session made the workload a process-group leader, so its pgid + # is its own pid. Asserting against that, rather than re-running the + # production expression os.getpgid(pid), keeps the expected value + # independent of the code under test. + assert result == workload.pid + assert scan.call_count == 2, "the scan was not retried after it raised" + # AND nothing was logged at all. The two debug lines on this path sit below + # the level build_logger sets, so an empty queue says more than the absence + # of one particular string -- which a reworded warning would satisfy + # silently. + assert collect_queue_messages(message_queue) == [] + + def test_scan_that_always_fails_times_out_and_reports_the_last_error( + self, + sleeper: Callable[..., Popen], + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + """Retrying is not the same as retrying forever. + + A scan that never succeeds must still end at the timeout, with None and + the warning -- and the warning must carry the last scan error, because + with the error no longer aborting the search it is otherwise discarded + and the timeout message alone cannot say what kept going wrong. + """ + # GIVEN + sudo_process = sleeper() + logger = build_logger(queue_handler) + # The wording find_child_process_id_pgrep really uses for this, so that a + # reader is not left wondering whether it was invented. + scan_error = "Expected a single child process of sudo, but found ['4321', '4322']" + scan = Mock(side_effect=FindSignalTargetError(scan_error)) + + # WHEN + with ( + patch("openjd.sessions._linux._sudo.find_child_process_id_pgrep", scan), + patch("openjd.sessions._linux._sudo.is_linux", return_value=False), + ): + result = find_sudo_child_process_group_id( + logger=logger, + sudo_process=sudo_process, + # A whole second rather than a fraction: the assertion below needs a + # second iteration to have begun and each one sleeps 0.05s, so a + # tight budget would turn a single overrunning sleep on a loaded + # host into a failure. The scan never succeeds, so the full window + # is spent either way. + timeout_seconds=1.0, + ) + + # THEN + assert result is None + assert scan.call_count > 1, "the scan was not retried after it raised" + messages = collect_queue_messages(message_queue) + warnings = [m for m in messages if "Unable to determine signal target" in m] + assert len(warnings) == 1, messages + # The timeout is what is reported, not the individual scan failure: that + # distinction is the whole difference between the fixed and broken code. + assert "unable to detect subprocess before timeout" in warnings[0] + assert f"last scan error: {scan_error}" in warnings[0] + + +@pytest.mark.skipif(not is_posix(), reason="pgrep and process groups are posix-only") +class TestFindSudoChildProcessGroupIdProcessGroupRaces: + """Tests of how find_sudo_child_process_group_id() resolves the two races that + are not scan failures: the child exiting before its group can be read, and the + child not having left sudo's process group yet. + + Both behaviours predate the retry-scoping fix above and had to survive it, so + they are pinned separately from it. The scan is mocked for the same reason as + above -- these are races, and a test that waited for one to occur naturally + would be a test that usually skipped the interesting path. + """ + + def test_child_that_exits_mid_scan_still_short_circuits( + self, + sleeper: Callable[..., Popen], + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + """A child that is gone by the time its group is read is answered + immediately with None. + + Retrying that would burn the whole timeout window on a process that is + known to be gone, and would report it as a timeout rather than as the + ordinary "it already exited" it is -- which is why no warning is logged. + """ + # GIVEN a pid that has been reaped, so os.getpgid() raises for it + departed = sleeper(own_process_group=True) + departed.kill() + departed.wait() + with pytest.raises(ProcessLookupError): + os.getpgid(departed.pid) # type: ignore + sudo_process = sleeper() + logger = build_logger(queue_handler) + scan = Mock(return_value=departed.pid) + + # WHEN + with ( + patch("openjd.sessions._linux._sudo.find_child_process_id_pgrep", scan), + patch("openjd.sessions._linux._sudo.is_linux", return_value=False), + ): + result = find_sudo_child_process_group_id( + logger=logger, + sudo_process=sudo_process, + timeout_seconds=5.0, + ) + + # THEN + assert result is None + assert scan.call_count == 1, "an exited child was scanned for again" + # AND nothing was logged: this is an ordinary outcome, not a failure to + # report. Asserted as an empty queue rather than as the absence of one + # string, which a reworded warning would satisfy silently. + assert collect_queue_messages(message_queue) == [] + + def test_child_still_sharing_sudos_process_group_is_never_accepted( + self, + sleeper: Callable[..., Popen], + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + """sudo's own process group is not an answer. + + Accepting it would aim a later SIGKILL at the group containing this + process, so a cancel would take down the session runtime along with the + workload. Here the group never changes, so the loop must exhaust the + timeout rather than accept what it has. + """ + # GIVEN a live process in the same process group as the stand-in for sudo + sudo_process = sleeper() + sibling = sleeper() + assert os.getpgid(sibling.pid) == os.getpgid( # type: ignore + sudo_process.pid + ), "test setup: the two processes were expected to share a process group" + logger = build_logger(queue_handler) + scan = Mock(return_value=sibling.pid) + + # WHEN + with ( + patch("openjd.sessions._linux._sudo.find_child_process_id_pgrep", scan), + patch("openjd.sessions._linux._sudo.is_linux", return_value=False), + ): + result = find_sudo_child_process_group_id( + logger=logger, + sudo_process=sudo_process, + timeout_seconds=0.2, + ) + + # THEN + assert result is None + messages = collect_queue_messages(message_queue) + warnings = [m for m in messages if "Unable to determine signal target" in m] + assert len(warnings) == 1, messages + assert "unable to detect subprocess before timeout" in warnings[0] + # AND no scan error is invented: nothing here failed to scan. + assert "last scan error" not in warnings[0] + + def test_child_is_accepted_once_it_leaves_sudos_process_group( + self, + tmp_path: Path, + sleeper: Callable[..., Popen], + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + """The other side of the same behaviour: sharing sudo's group is a reason + to retry, not to give up. + + The child moves into its own group partway through the retry window, as + sudo's child really does, and the pgid it reports for itself is what the + function must return. + """ + # GIVEN + pgid_file = tmp_path / "child_pgid.txt" + sudo_process = sleeper() + logger = build_logger(queue_handler) + # Started without own_process_group, so it begins life in this process' + # group -- the same group as the stand-in for sudo -- and calls setsid() + # only after the delay. Registered with the fixture, so it is reaped even + # if the setup assertion below fails. + child = sleeper( + script=LATE_PROCESS_GROUP_SCRIPT + % {"pgid_file": repr(str(pgid_file)), "delay_seconds": 0.5} + ) + scan = Mock(return_value=child.pid) + + # The delay gives this assertion ample room: the child has to start a + # Python interpreter and sleep 0.5s before it can change groups. + assert os.getpgid(child.pid) == os.getpgid( # type: ignore + sudo_process.pid + ), "test setup: the child was expected to start in sudo's process group" + + # WHEN + with ( + patch("openjd.sessions._linux._sudo.find_child_process_id_pgrep", scan), + patch("openjd.sessions._linux._sudo.is_linux", return_value=False), + ): + result = find_sudo_child_process_group_id( + logger=logger, + sudo_process=sudo_process, + timeout_seconds=10.0, + ) + + # THEN + reported_pgid = poll_for_pgid(pgid_file, timeout_seconds=5.0) + assert reported_pgid is not None, ( + "the child never reported a new process group, so this test could not have " + f"observed one (exit code: {child.poll()}, stderr: {stderr_of(child)!r})" + ) + assert result == reported_pgid + assert result != os.getpgid(sudo_process.pid) # type: ignore + # AND nothing was logged: retrying past the shared group is not a failure. + assert collect_queue_messages(message_queue) == [] diff --git a/test/openjd/sessions_v0/test_optimized_mode_invariants.py b/test/openjd/sessions_v0/test_optimized_mode_invariants.py new file mode 100644 index 00000000..f7dc7c38 --- /dev/null +++ b/test/openjd/sessions_v0/test_optimized_mode_invariants.py @@ -0,0 +1,145 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Runtime invariants must survive ``python -O``, which strips every ``assert``. + +Checks that carry a real invariant -- as opposed to type-checker narrowing -- +have to be explicit raises. See the policy in DEVELOPMENT.md. +""" + +import sys +from datetime import timedelta +from pathlib import Path +from typing import Optional +from unittest.mock import MagicMock + +import pytest + +from openjd.model import SymbolTable +from openjd.model.v2023_09 import ( + DataString as DataString_2023_09, + EmbeddedFileText as EmbeddedFileText_2023_09, + EmbeddedFileTypes as EmbeddedFileTypes_2023_09, +) + +from openjd.sessions import Session +from openjd.sessions._embedded_files import ( + EmbeddedFiles, + EmbeddedFilesScope, + _FileRecord, +) +from openjd.sessions._runner_base import ScriptRunnerBase +from openjd.sessions._subprocess import LoggingSubprocess + + +class _Runner(ScriptRunnerBase): + """Minimal concrete runner: only _generate_command_shell_script is exercised.""" + + def cancel( + self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False + ) -> None: + pass + + +class TestInvariantsSurviveOptimizedMode: + """R5-6: `assert` is stripped under `python -O`, which is a legitimate + production deployment mode. Checks that carry a real runtime invariant must + be explicit raises.""" + + def test_unsupported_embedded_file_model_raises(self, tmp_path: Path) -> None: + # GIVEN: an embedded file object of an unrecognised shape + files = EmbeddedFiles( + logger=MagicMock(), + scope=EmbeddedFilesScope.ENV, + session_files_directory=tmp_path, + ) + + class _AlienFile: + filename = "x.sh" + runnable = False + + # WHEN / THEN: each of the three entry points stops rather than + # materializing a file from an object it does not understand. + with pytest.raises(RuntimeError, match="Unsupported embedded file model"): + files._find_value_prefix(_AlienFile()) # type: ignore[arg-type] + with pytest.raises(RuntimeError, match="Unsupported embedded file model"): + files._get_symtab_entry(_AlienFile()) # type: ignore[arg-type] + with pytest.raises(RuntimeError, match="Unsupported embedded file model"): + files._materialize_file( + tmp_path / "x.sh", _AlienFile(), SymbolTable() # type: ignore[arg-type] + ) + + def test_working_directory_property_raises_a_legible_error(self) -> None: + # GIVEN: a Session whose construction did not complete + session = Session.__new__(Session) + session._working_dir = None # type: ignore[attr-defined] + + # WHEN / THEN: a library RuntimeError, not AttributeError on None. + with pytest.raises(RuntimeError, match="no working directory"): + _ = session.working_directory + + def test_state_is_readable_when_a_launch_failed_after_process_creation( + self, tmp_path: Path + ) -> None: + """The `state` property is on the path every consumer polls. An + inconsistent (process set, future unset) pair must not make the runner + permanently unreadable.""" + # GIVEN + runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) + try: + runner._process = MagicMock() + runner._run_future = None + + # WHEN / THEN: no AssertionError, no AttributeError. + from openjd.sessions._runner_base import ScriptRunnerState + + assert runner.state == ScriptRunnerState.READY + finally: + runner.shutdown() + + def test_notify_period_end_is_a_no_op_without_a_process(self, tmp_path: Path) -> None: + """Runs on a threading.Timer thread, where an exception reaches only + threading.excepthook -- so the grace period would appear to have silently + done nothing.""" + # GIVEN + runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) + try: + runner._process = None + + # WHEN / THEN: returns cleanly. + runner._on_notify_period_end() + finally: + runner.shutdown() + + def test_log_subproc_stdout_raises_a_legible_error_without_a_process( + self, tmp_path: Path + ) -> None: + # GIVEN + proc = LoggingSubprocess(logger=MagicMock(), args=[sys.executable, "-c", "pass"]) + + # WHEN / THEN + with pytest.raises(RuntimeError, match="before the subprocess has been created"): + proc._log_subproc_stdout() + + def test_an_embedded_file_still_materializes_normally(self, tmp_path: Path) -> None: + """Guard against the isinstance rewrite breaking the happy path.""" + # GIVEN + files = EmbeddedFiles( + logger=MagicMock(), + scope=EmbeddedFilesScope.ENV, + session_files_directory=tmp_path, + ) + model = EmbeddedFileText_2023_09( + name="Script", + type=EmbeddedFileTypes_2023_09.TEXT, + data=DataString_2023_09("hello"), + ) + symtab = SymbolTable() + + # WHEN + records = files.allocate_file_paths([model], symtab) + files.write_file_contents(records, symtab) + + # THEN + assert len(records) == 1 + assert isinstance(records[0], _FileRecord) + assert records[0].filename.read_text() == "hello" diff --git a/test/openjd/sessions_v0/test_path_mapping.py b/test/openjd/sessions_v0/test_path_mapping.py index 434ab517..73527b5d 100644 --- a/test/openjd/sessions_v0/test_path_mapping.py +++ b/test/openjd/sessions_v0/test_path_mapping.py @@ -535,3 +535,179 @@ def test_from_dict_success(self, dict_rule, expected): def test_from_dict_failure(self, dict_rule): with pytest.raises(ValueError): PathMappingRule.from_dict(dict_rule) + + +class TestUriPathMapping: + """RFC 0006 §2.3.2 (EXPR): URI-form source paths. + + Per RFC 3986 the scheme and authority match case-insensitively while the + path portion matches case-sensitively, on whole path components. openjd-rs + implements the prefix comparison with ``str::eq_ignore_ascii_case``, so the + fold must be ASCII-only here too. + """ + + def _rule(self, source: str, destination: str = "/local") -> PathMappingRule: + # Construct directly with a PurePosixPath destination rather than going + # through from_dict, which builds a host-flavoured PurePath -- on Windows + # that renders "/local" as "\\local" and the expectations below would + # have to be host-dependent for no reason. os_name is patched to posix in + # each test, so the child separator is "/" on both hosts. + return PathMappingRule( + source_path_format=PathFormat.URI, + source_path=source, + destination_path=PurePosixPath(destination), + ) + + @pytest.mark.parametrize( + "given, expected", + [ + pytest.param("s3://bucket/prefix/f.obj", (True, "/local/f.obj"), id="child-file"), + pytest.param("s3://bucket/prefix", (True, "/local"), id="exact-source"), + pytest.param("S3://BUCKET/prefix/f.obj", (True, "/local/f.obj"), id="ascii-fold"), + pytest.param( + "s3://bucket/PREFIX/f.obj", + (False, "s3://bucket/PREFIX/f.obj"), + id="path-is-case-sensitive", + ), + pytest.param( + "s3://other/prefix/f.obj", + (False, "s3://other/prefix/f.obj"), + id="different-authority", + ), + pytest.param( + "s3://bucket/prefixed/f.obj", + (False, "s3://bucket/prefixed/f.obj"), + id="not-a-component-boundary", + ), + ], + ) + def test_apply_uri(self, given: str, expected: tuple[bool, str]) -> None: + # GIVEN + rule = self._rule("s3://bucket/prefix") + + # WHEN / THEN + with patch.object(path_mapping_impl_mod, "os_name", OSName.POSIX.value): + assert rule.apply(path=given) == expected + + def test_scheme_authority_fold_is_ascii_only(self) -> None: + """A non-ASCII authority must not fold to an ASCII one. + + U+212A KELVIN SIGN lower-cases to ASCII 'k' under ``str.lower()``, so a + Unicode fold would make these two authorities match — where openjd-rs's + ``eq_ignore_ascii_case`` keeps them distinct. + """ + # GIVEN + rule = self._rule("s3://bucket\u212a") + + # WHEN + with patch.object(path_mapping_impl_mod, "os_name", OSName.POSIX.value): + matched, result = rule.apply(path="s3://bucketk/f.obj") + + # THEN + assert matched is False + assert result == "s3://bucketk/f.obj" + + def test_non_ascii_authority_still_matches_itself(self) -> None: + # GIVEN + rule = self._rule("s3://bucket\u212a") + + # WHEN + with patch.object(path_mapping_impl_mod, "os_name", OSName.POSIX.value): + matched, result = rule.apply(path="S3://BUCKET\u212a/f.obj") + + # THEN: ASCII letters fold, the non-ASCII character compares as itself + assert matched is True + assert result == "/local/f.obj" + + +class TestWindowsAsciiOnlyFold: + """Windows path rules match case-insensitively, but over ASCII only. + + `PureWindowsPath.is_relative_to()` folds with `str.lower()`, which also folds + non-ASCII characters, so a homoglyph in a submitted path would remap here + while the EXPR engine's `apply_path_mapping()` and openjd-rs -- both using an + ASCII-only fold -- leave it alone. RFC 0006 2.3.2 says the two are the same + transformation. + """ + + def _rule(self, source: str) -> PathMappingRule: + return PathMappingRule( + source_path_format=PathFormat.WINDOWS, + source_path=PureWindowsPath(source), + destination_path=PurePosixPath("/mnt/assets"), + ) + + def test_non_ascii_homoglyph_does_not_match(self) -> None: + # GIVEN: a rule whose source ends in ASCII 'k' + rule = self._rule("C:\\assets\\k") + + # WHEN: the input uses U+212A KELVIN SIGN, which str.lower() folds to 'k' + with patch.object(path_mapping_impl_mod, "os_name", OSName.POSIX.value): + matched, result = rule.apply(path="C:\\assets\\\u212a\\shot.exr") + + # THEN: no match, matching the engine and openjd-rs + assert matched is False + assert result == "C:\\assets\\\u212a\\shot.exr" + + @pytest.mark.parametrize( + "given, expected", + [ + pytest.param("C:\\assets\\shot.exr", (True, "/mnt/assets/shot.exr"), id="plain"), + pytest.param( + "C:\\ASSETS\\shot.exr", (True, "/mnt/assets/shot.exr"), id="ascii-fold-still-works" + ), + pytest.param( + "C:/assets/scenes/", (True, "/mnt/assets/scenes/"), id="forward-slash-trailing" + ), + pytest.param("C:\\assets\\", (True, "/mnt/assets/"), id="backslash-trailing"), + pytest.param( + "D:\\assets\\shot.exr", (False, "D:\\assets\\shot.exr"), id="different-drive" + ), + pytest.param( + "C:\\assetsmore\\x", (False, "C:\\assetsmore\\x"), id="not-a-component-boundary" + ), + ], + ) + def test_windows_matching(self, given: str, expected: tuple[bool, str]) -> None: + # GIVEN + rule = self._rule("C:\\assets") + + # WHEN / THEN + with patch.object(path_mapping_impl_mod, "os_name", OSName.POSIX.value): + assert rule.apply(path=given) == expected + + +class TestUriRuleValidation: + """A URI-format rule's source_path must carry a real scheme. + + The EXPR engine requires one, and these rules are handed to it via the + session's host context -- so a malformed rule has to be rejected where the + offending value can still be named, not later from inside `Session()`. + """ + + @pytest.mark.parametrize( + "source", + ["s3:/bucket", "notauri", "://bucket", "1s3://bucket", "", "s_3://bucket"], + ) + def test_malformed_uri_source_rejected(self, source: str) -> None: + with pytest.raises(ValueError, match="must begin with"): + PathMappingRule.from_dict( + rule={ + "source_path_format": "URI", + "source_path": source, + "destination_path": "/local", + } + ) + + @pytest.mark.parametrize( + "source", ["s3://bucket/a", "file:///x", "nfs://host:2049/export", "S3://Bucket"] + ) + def test_well_formed_uri_source_accepted(self, source: str) -> None: + rule = PathMappingRule.from_dict( + rule={ + "source_path_format": "URI", + "source_path": source, + "destination_path": "/local", + } + ) + assert rule.source_path == source diff --git a/test/openjd/sessions_v0/test_runner_base.py b/test/openjd/sessions_v0/test_runner_base.py index 234eb999..4a993b42 100644 --- a/test/openjd/sessions_v0/test_runner_base.py +++ b/test/openjd/sessions_v0/test_runner_base.py @@ -8,7 +8,7 @@ from pathlib import Path from queue import SimpleQueue from typing import Optional, cast -from unittest.mock import MagicMock, call +from unittest.mock import MagicMock, call, patch import pytest @@ -30,6 +30,7 @@ from openjd.sessions._os_checker import is_posix, is_windows from openjd.sessions._runner_base import ( + MAX_INT_FIELD_VALUE, NotifyCancelMethod, ScriptRunnerBase, ScriptRunnerState, @@ -38,6 +39,7 @@ from openjd.sessions._tempdir import TempDir from .conftest import ( + serial_process, build_logger, collect_queue_messages, has_posix_target_user, @@ -608,6 +610,7 @@ def test_cannot_run_twice(self, tmp_path: Path, python_exe: str) -> None: with pytest.raises(RuntimeError): runner._run([python_exe, "-c", "print('hello')"]) + @serial_process @pytest.mark.usefixtures("message_queue", "queue_handler") def test_run_action( self, @@ -649,33 +652,43 @@ def test_run_action( assert "Log from test 0" in messages assert "Log from test 9" not in messages - @pytest.mark.usefixtures("message_queue", "queue_handler") @pytest.mark.parametrize( - argnames=("default_timeout_seconds", "action_timeout_seconds"), + argnames=("default_timeout_seconds", "action_timeout_seconds", "expected_seconds"), argvalues=( - pytest.param(1, 5, id="action-timeout-prevails"), - pytest.param(2, None, id="default-applied"), - pytest.param(None, None, id="no-timeout"), + pytest.param(1, 5, 5, id="action-timeout-prevails"), + pytest.param(5, 1, 1, id="action-timeout-prevails-when-smaller"), + pytest.param(2, None, 2, id="default-applied"), + pytest.param(None, 7, 7, id="action-only"), + pytest.param(None, None, None, id="no-timeout"), ), ) - def test_run_action_default_timeout( + def test_run_action_effective_timeout( self, tmp_path: Path, - message_queue: SimpleQueue, - queue_handler: QueueHandler, default_timeout_seconds: Optional[int], action_timeout_seconds: Optional[int], + expected_seconds: Optional[int], python_exe: str, ) -> None: - # Tests that the effective timeout is applied correctly given a supplied default timeout - # and an optional timeout defined on the action - + """The effective time limit is the action's timeout if it has one, else the + caller's default, else none at all. + + Asserts the time limit `_run_action` hands to `_run`, with no subprocess + and no wall clock involved. + + This replaces an end-to-end version that started a child printing one line + per second and asserted a +/-1 second window on its output + (`"Log from test {T-1}" in messages`). That coupled three unrelated + things -- timeout *selection*, `threading.Timer` scheduling, and child + process startup latency -- and only the first is what this test is named + for. The Timer is armed before the child is submitted to the pool, so the + Timer's clock starts before the interpreter even launches; any host slow + enough to delay startup past a second failed the assertion while the + product behaved correctly. Measured: injecting a 1.5s startup delay + against a 2s timeout fails the old assertion with the runner correctly in + TIMEOUT. It was the suite's most persistent false red. + """ # GIVEN - expected_effective_timeout_seconds: Optional[int] = None - if action_timeout_seconds is not None: - expected_effective_timeout_seconds = action_timeout_seconds - elif default_timeout_seconds is not None: - expected_effective_timeout_seconds = default_timeout_seconds default_timeout = ( timedelta(seconds=default_timeout_seconds) if default_timeout_seconds is not None @@ -683,9 +696,50 @@ def test_run_action_default_timeout( ) action = Action_2023_09( command=CommandString_2023_09("{{Task.PythonInterpreter}}"), - args=[ArgString_2023_09("{{Task.ScriptFile}}")], + args=[ArgString_2023_09("-c"), ArgString_2023_09("pass")], timeout=action_timeout_seconds, ) + symtab = SymbolTable(source={"Task.PythonInterpreter": python_exe}) + captured: list[Optional[timedelta]] = [] + + with TerminatingRunner(logger=MagicMock(), session_working_directory=tmp_path) as runner: + # WHEN + with patch.object( + runner, + "_run", + side_effect=lambda args, time_limit=None: captured.append(time_limit), + ): + runner._run_action(action, symtab, default_timeout=default_timeout) + + # THEN + assert len(captured) == 1, "the action was not launched exactly once" + expected = timedelta(seconds=expected_seconds) if expected_seconds is not None else None + assert captured[0] == expected + + @serial_process + @pytest.mark.usefixtures("message_queue", "queue_handler") + def test_run_action_timeout_terminates_the_action( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + python_exe: str, + ) -> None: + """A declared timeout really does terminate the action. + + The end-to-end half of the coverage above, kept deliberately loose: the + terminal state, and that the child did not reach its last line. It says + nothing about *when* the timeout landed, because which second the child + reaches depends on how promptly the host scheduled it -- that assumption is + what made the previous version of this test flaky. + """ + # GIVEN: a child that prints one line a second for 20 seconds, and a + # timeout that will cut it short + action = Action_2023_09( + command=CommandString_2023_09("{{Task.PythonInterpreter}}"), + args=[ArgString_2023_09("{{Task.ScriptFile}}")], + timeout=2, + ) python_app_loc = (Path(__file__).parent / "support_files" / "app_20s_run.py").resolve() symtab = SymbolTable( source={ @@ -696,25 +750,131 @@ def test_run_action_default_timeout( logger = build_logger(queue_handler) with TerminatingRunner(logger=logger, session_working_directory=tmp_path) as runner: # WHEN - runner._run_action(action, symtab, default_timeout=default_timeout) - # wait for the process to exit + runner._run_action(action, symtab) while runner.state == ScriptRunnerState.RUNNING: time.sleep(0.2) # THEN - if expected_effective_timeout_seconds is not None: - assert runner.state == ScriptRunnerState.TIMEOUT - else: + assert runner.state == ScriptRunnerState.TIMEOUT + assert "Log from test 19" not in collect_queue_messages(message_queue) + + @pytest.mark.usefixtures("message_queue", "queue_handler") + def test_run_action_without_timeout_runs_to_completion( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + python_exe: str, + ) -> None: + """No timeout means no time limit is imposed: the action finishes on its + own terms. + + Uses a child that exits after a fraction of a second rather than the + 20-second one. Running a 20-second child to completion made this the single + slowest test in the suite by a factor of three, and it was the whole + critical path under `-n auto` -- while proving nothing that a short child + does not. What is being asserted is that no timer cut the action short, and + a child that exits on its own shows that either way. + """ + # GIVEN: a short-lived child, and no timeout on the action or the caller + action = Action_2023_09( + command=CommandString_2023_09("{{Task.PythonInterpreter}}"), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09("import time; time.sleep(0.5); print('finished')"), + ], + timeout=None, + ) + symtab = SymbolTable(source={"Task.PythonInterpreter": python_exe}) + logger = build_logger(queue_handler) + with TerminatingRunner(logger=logger, session_working_directory=tmp_path) as runner: + # WHEN + runner._run_action(action, symtab, default_timeout=None) + while runner.state == ScriptRunnerState.RUNNING: + time.sleep(0.05) + + # THEN: it ran to its own end, and its last output arrived. + assert runner.state == ScriptRunnerState.SUCCESS + assert runner.exit_code == 0 + assert "finished" in collect_queue_messages(message_queue) + + @pytest.mark.usefixtures("message_queue", "queue_handler") + @pytest.mark.parametrize( + argnames="timeout_seconds", + argvalues=( + # Larger than datetime.timedelta can represent. + pytest.param(86_400_000_000_000, id="over-timedelta-max"), + # Representable by timedelta, but threading.Timer's deadline + # arithmetic overflows CPython's 64-bit nanosecond time + # representation. + pytest.param(86_399_999_913_600, id="over-schedulable-but-in-timedelta"), + pytest.param(9_223_372_036_854_775_807, id="i64-max"), + ), + ) + def test_run_action_unenforceable_timeout_runs_unbounded( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + timeout_seconds: int, + ) -> None: + """A timeout too large to schedule must not raise; the action runs. + + The 2023-09 schema puts no upper bound on `timeout`, so these + values are template-legal. openjd-rs resolves them with + `Duration::from_secs` and runs the action, so we do the same rather than + raising OverflowError out of the public Session API (which used to leave + the Session stuck in RUNNING with no terminal ActionStatus). + """ + # GIVEN + action = Action_2023_09( + command=CommandString_2023_09("{{Task.Command}}"), + args=[ArgString_2023_09("ok")], + timeout=timeout_seconds, + ) + symtab = SymbolTable(source={"Task.Command": "echo" if is_posix() else "cmd.exe"}) + logger = build_logger(queue_handler) + + # WHEN + with TerminatingRunner(logger=logger, session_working_directory=tmp_path) as runner: + runner._run_action(action, symtab) + while runner.state == ScriptRunnerState.RUNNING: + time.sleep(0.2) + + # THEN: it ran, with no time limit scheduled assert runner.state == ScriptRunnerState.SUCCESS + assert runner._runtime_limit is None messages = collect_queue_messages(message_queue) - # The application prints out 0, ..., 19 once a second for 20s . - # If it ended early, then we printed the first but not the last. - print(messages) - if expected_effective_timeout_seconds is not None: - assert f"Log from test {expected_effective_timeout_seconds - 1}" in messages - assert f"Log from test {expected_effective_timeout_seconds + 1}" not in messages - else: - assert "Log from test 19" in messages + assert any("larger than this runtime can enforce" in m for m in messages) + + @pytest.mark.usefixtures("queue_handler") + def test_run_action_over_range_timeout_fails_action( + self, + tmp_path: Path, + queue_handler: QueueHandler, + ) -> None: + """An over-range timeout fails the action, matching openjd-rs. + + openjd-rs rejects a literal above i64::MAX at parse time and its runtime + parses a resolved one with str::parse, which fails rather than + saturating -- so the action must fail through the normal failure path + instead of running. See MAX_INT_FIELD_VALUE. + """ + # GIVEN + action = Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("ok")], + timeout=MAX_INT_FIELD_VALUE + 1, + ) + logger = build_logger(queue_handler) + + # WHEN + with TerminatingRunner(logger=logger, session_working_directory=tmp_path) as runner: + runner._run_action(action, SymbolTable()) + + # THEN + assert runner.state == ScriptRunnerState.FAILED + assert runner._runtime_limit is None @pytest.mark.usefixtures("message_queue", "queue_handler") def test_run_action_bad_formatstring( @@ -744,6 +904,7 @@ def test_run_action_bad_formatstring( messages = collect_queue_messages(message_queue) assert any(m.startswith("openjd_fail") for m in messages) + @serial_process @pytest.mark.usefixtures("message_queue", "queue_handler") def test_cancel_terminate( self, @@ -779,6 +940,7 @@ def test_cancel_terminate( # Didn't get to the end of the application run assert "Log from test 9" not in messages + @serial_process @pytest.mark.usefixtures("message_queue", "queue_handler") @pytest.mark.xfail(not is_posix(), reason="Signals not yet implemented for non-posix") def test_run_with_time_limit( @@ -811,6 +973,7 @@ def test_run_with_time_limit( # Didn't get to the end of the application run assert "Log from test 9" not in messages + @serial_process @pytest.mark.usefixtures("message_queue", "queue_handler") def test_cancel_notify( self, @@ -974,6 +1137,7 @@ def test_cancel_notify_direct_signal_with_cap_kill( cap_kill_was_effective == cap_kill_effective_after_cancel ), "CAP_KILL added/removed from effetive set and persisted after cancelation" + @serial_process @pytest.mark.usefixtures("message_queue", "queue_handler") def test_cancel_double_cancel_notify( self, diff --git a/test/openjd/sessions_v0/test_runner_env_script.py b/test/openjd/sessions_v0/test_runner_env_script.py index 586f08e6..c06cdb27 100644 --- a/test/openjd/sessions_v0/test_runner_env_script.py +++ b/test/openjd/sessions_v0/test_runner_env_script.py @@ -35,13 +35,13 @@ DataString as DataString_2023_09, ) from openjd.sessions import ActionState -from openjd.sessions._runner_base import ScriptRunnerState -from openjd.sessions._runner_env_script import ( +from openjd.sessions._runner_base import ( CancelMethod, - EnvironmentScriptRunner, NotifyCancelMethod, + ScriptRunnerState, TerminateCancelMethod, ) +from openjd.sessions._runner_env_script import EnvironmentScriptRunner from .conftest import build_logger, collect_queue_messages @@ -387,7 +387,9 @@ def test_cancel( # The lower-level process runners have been thoroughly tested for cancel's # functionality, so this seems fine. - with patch.object(EnvironmentScriptRunner, "_run_action"): + # Patch _run (not _run_action): the effective cancel method is now + # resolved by _run_action at launch time and consumed by cancel(). + with patch.object(EnvironmentScriptRunner, "_run"): with patch.object(EnvironmentScriptRunner, "_cancel") as mock_cancel: # GIVEN script = EnvironmentScript_2023_09( @@ -409,6 +411,10 @@ def test_cancel( session_files_directory=tmp_path, ) runner.enter() + # _run is patched out, so stand in for the subprocess it would have + # created: a cancel that arrives before one exists is deferred + # rather than applied (see ScriptRunnerBase._pending_cancel). + runner._process = MagicMock() time_limit = timedelta(30) # WHEN @@ -468,6 +474,7 @@ def test_run_env_action_passes_default_timeout( action, symtab, default_timeout=default_timeout, + default_notify_period_seconds=30, ) def test_exit_uses_default_timeout( diff --git a/test/openjd/sessions_v0/test_runner_launch_state.py b/test/openjd/sessions_v0/test_runner_launch_state.py new file mode 100644 index 00000000..3ed6d25d --- /dev/null +++ b/test/openjd/sessions_v0/test_runner_launch_state.py @@ -0,0 +1,203 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Runner state around a launch that fails after the subprocess object exists. + +``_run``’s single-use guard and the terminal state it publishes both have to hold +when the process/future pair is left inconsistent. +""" + +import sys +import threading +from concurrent.futures import Future, ThreadPoolExecutor +from datetime import timedelta +from pathlib import Path +from typing import Any, Optional +from unittest.mock import MagicMock, patch + +import pytest + +from openjd.sessions import ActionState +from openjd.sessions._runner_base import ScriptRunnerBase, ScriptRunnerState + + +class _Runner(ScriptRunnerBase): + """Minimal concrete runner: only _generate_command_shell_script is exercised.""" + + def cancel( + self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False + ) -> None: + pass + + +class TestFailedLaunchDoesNotCorruptRunnerState: + """A launch that fails after the subprocess object exists must not leave the + runner looking reusable, and a completed action must always publish a + terminal state. + + `_run` gates re-entry on `state == READY`. When R5-6 made `state` report + READY for the (process set, future unset) pair, that guard silently opened. + And `_on_process_exit` built `ActionState(self.state.value)`, which raises + `ValueError` for 'ready' -- swallowed by the F8 try/except, so the consumer + received no terminal callback at all and would wait forever. + """ + + def _runner_with_dead_pool(self, tmp_path: Path, published: list) -> _Runner: + runner = _Runner( + logger=MagicMock(), + session_working_directory=tmp_path, + callback=lambda s: published.append(s.value), + ) + # Exactly what Session.cleanup() does to the runner's pool. A cleanup + # racing a launch is the ordinary agent SIGTERM path. + runner._pool.shutdown(wait=True) + return runner + + def test_second_launch_is_refused_after_a_failed_launch(self, tmp_path: Path) -> None: + # GIVEN: a runner whose first launch fails inside _pool.submit + published: list = [] + runner = self._runner_with_dead_pool(tmp_path, published) + try: + with pytest.raises(RuntimeError): + runner._run([sys.executable, "-c", "pass"], timedelta(seconds=600)) + + # WHEN: the runner is handed a working pool and asked to run again, + # exactly as a caller that caught the RuntimeError might. + runner._pool = ThreadPoolExecutor(max_workers=1) + + # THEN: the single-use guard holds. Before the fix this launched a + # second real child and replaced _process, orphaning the first. + with pytest.raises(RuntimeError, match="cannot be used to run a second subprocess"): + runner._run([sys.executable, "-c", "import time; time.sleep(30)"], None) + finally: + runner.cancel() + for timer in [t for t in threading.enumerate() if isinstance(t, threading.Timer)]: + timer.cancel() + runner.shutdown() + + def test_launch_latch_holds_even_when_state_reports_ready(self, tmp_path: Path) -> None: + """Pins the guard to `_launched` rather than to the state classification. + + This is the property that actually broke: whatever `state` decides to + report for an inconsistent runner, re-entry must still be refused. + """ + # GIVEN: a runner that has attempted a launch, with `state` forced to + # report READY (the exact misclassification that opened the guard) + runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) + try: + runner._pool.shutdown(wait=True) + with pytest.raises(RuntimeError): + runner._run([sys.executable, "-c", "pass"], None) + runner._pool = ThreadPoolExecutor(max_workers=1) + + with patch.object( + type(runner), "state", property(lambda self: ScriptRunnerState.READY) + ): + assert runner.state == ScriptRunnerState.READY # the hostile premise + # THEN + with pytest.raises(RuntimeError, match="second subprocess"): + runner._run([sys.executable, "-c", "pass"], None) + finally: + runner.shutdown() + + def test_completion_always_publishes_a_terminal_state(self, tmp_path: Path) -> None: + # GIVEN: a runner whose process/future pair is inconsistent, so `state` + # cannot be classified as a terminal ActionState + published: list = [] + runner = _Runner( + logger=MagicMock(), + session_working_directory=tmp_path, + callback=lambda s: published.append(s), + ) + try: + runner._process = MagicMock() + runner._run_future = None + completed: Future = Future() + completed.set_result(None) + + # WHEN + runner._on_process_exit(completed) + + # THEN: exactly one terminal callback. Before the fix this list was + # empty: ActionState('ready') raised and the F8 handler ate it. + assert len(published) == 1 + assert published[0] in ( + ActionState.FAILED, + ActionState.CANCELED, + ActionState.TIMEOUT, + ActionState.SUCCESS, + ) + assert published[0] == ActionState.FAILED + finally: + runner.shutdown() + + def test_terminal_action_state_maps_every_runner_state(self, tmp_path: Path) -> None: + """No ScriptRunnerState may make the terminal mapping raise.""" + # GIVEN + runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) + try: + for runner_state in ScriptRunnerState: + + def fixed_state(_self: Any, _s: ScriptRunnerState = runner_state) -> Any: + return _s + + # WHEN + with patch.object(type(runner), "state", property(fixed_state)): + result = runner._terminal_action_state() + # THEN + assert isinstance(result, ActionState) + finally: + runner.shutdown() + + def test_process_and_future_are_published_together(self, tmp_path: Path) -> None: + """Removes the root cause: a reader must never see a process with no + future, on the success path or any other.""" + # GIVEN: a poller hammering `state` from another thread during launch + observations: list[tuple[bool, bool]] = [] + stop = threading.Event() + runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) + + def poll() -> None: + while not stop.is_set(): + observations.append((runner._process is not None, runner._run_future is not None)) + + poller = threading.Thread(target=poll, daemon=True) + try: + poller.start() + # WHEN + runner._run([sys.executable, "-c", "pass"], None) + stop.set() + poller.join(timeout=10) + + # THEN: the (process, no future) combination was never observable. + assert (True, False) not in observations + assert observations, "the poller must actually have sampled something" + finally: + stop.set() + runner.shutdown() + + def test_a_normal_run_still_reports_success(self, tmp_path: Path) -> None: + """Guard against the latch or the restructure breaking the happy path.""" + # GIVEN + published: list = [] + runner = _Runner( + logger=MagicMock(), + session_working_directory=tmp_path, + callback=lambda s: published.append(s), + ) + try: + # WHEN + runner._run([sys.executable, "-c", "pass"], None) + assert runner._run_future is not None + runner._run_future.result(timeout=30) + + # THEN + deadline = 30.0 + import time + + start = time.monotonic() + while ActionState.SUCCESS not in published and time.monotonic() - start < deadline: + time.sleep(0.02) + assert runner.state == ScriptRunnerState.SUCCESS + assert ActionState.SUCCESS in published + finally: + runner.shutdown() diff --git a/test/openjd/sessions_v0/test_runner_step_script.py b/test/openjd/sessions_v0/test_runner_step_script.py index 2b258adf..9f3f4d9d 100644 --- a/test/openjd/sessions_v0/test_runner_step_script.py +++ b/test/openjd/sessions_v0/test_runner_step_script.py @@ -1,5 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +import threading import time from datetime import timedelta from logging.handlers import QueueHandler @@ -35,13 +36,14 @@ from openjd.model.v2023_09 import StepScript as StepScript_2023_09 from openjd.sessions import WindowsSessionUser -from openjd.sessions._runner_base import ScriptRunnerState -from openjd.sessions._runner_step_script import ( +from openjd.sessions._runner_base import ( CancelMethod, NotifyCancelMethod, - StepScriptRunner, + ScriptRunnerState, TerminateCancelMethod, ) +from openjd.sessions._runner_step_script import StepScriptRunner +from openjd.sessions._subprocess import LoggingSubprocess from openjd.sessions._tempdir import TempDir from openjd.sessions._os_checker import is_posix, is_windows @@ -252,7 +254,9 @@ def test_cancel( # The lower-level process runners have been thoroughly tested for cancel's # functionality, so this seems fine. - with patch.object(StepScriptRunner, "_run_action"): + # Patch _run (not _run_action): the effective cancel method is now + # resolved by _run_action at launch time and consumed by cancel(). + with patch.object(StepScriptRunner, "_run"): with patch.object(StepScriptRunner, "_cancel") as mock_cancel: # GIVEN script = StepScript_2023_09( @@ -273,6 +277,10 @@ def test_cancel( session_files_directory=tmp_path, ) runner.run() + # _run is patched out, so stand in for the subprocess it would have + # created: a cancel that arrives before one exists is deferred + # rather than applied (see ScriptRunnerBase._pending_cancel). + runner._process = MagicMock() time_limit = timedelta(30) # WHEN @@ -346,3 +354,161 @@ def test_run_file_in_session_dir_as_windows_user( assert runner.state == ScriptRunnerState.SUCCESS messages = collect_queue_messages(message_queue) assert "Hello!" in messages + + +class TestCancelRacingLaunch: + """A cancel is delivered from another thread, so it can land while the action + is still being set up -- after the effective cancel method has been resolved + but before the subprocess exists. It must be remembered and applied, not + dropped and not raised (openjd-rs holds the equivalent state in a sticky + CancellationToken).""" + + def _runner(self, tmp_path: Path, python_exe: str) -> StepScriptRunner: + script = StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09("print('hi')")], + ) + ) + ) + return StepScriptRunner( + logger=MagicMock(), + session_working_directory=tmp_path, + script=script, + symtab=SymbolTable(), + session_files_directory=tmp_path, + ) + + def test_cancel_before_subprocess_is_remembered(self, tmp_path: Path, python_exe: str) -> None: + # GIVEN: the action resolved (so a cancel method is stored) but _run was + # patched out, so no subprocess exists yet + with patch.object(StepScriptRunner, "_run"): + runner = self._runner(tmp_path, python_exe) + runner.run() + assert runner._process is None + assert runner._resolved_cancel_method == TerminateCancelMethod() + + # WHEN: a cancel lands in that window + runner.cancel(time_limit=timedelta(seconds=7)) + + # THEN: it is remembered rather than dropped, and nothing was raised + assert runner._pending_cancel == (timedelta(seconds=7), False) + + def test_pending_cancel_is_applied_once_the_subprocess_exists( + self, tmp_path: Path, python_exe: str + ) -> None: + # GIVEN: a cancel recorded during setup + with patch.object(StepScriptRunner, "_run"): + runner = self._runner(tmp_path, python_exe) + runner.run() + runner.cancel(mark_action_failed=True) + assert runner._pending_cancel == (None, True) + + # WHEN: the real _run finishes creating the subprocess + with patch.object(StepScriptRunner, "_cancel") as mock_cancel: + runner._process = MagicMock() + pending = runner._pending_cancel + assert pending is not None + runner._pending_cancel = None + assert runner._resolved_cancel_method is not None + runner._cancel(runner._resolved_cancel_method, *pending) + + # THEN: the stored method is delivered with the recorded arguments + assert mock_cancel.call_args.args[0] == TerminateCancelMethod() + assert mock_cancel.call_args.args[2] is True + + def test_cancel_with_no_subprocess_does_not_raise( + self, tmp_path: Path, python_exe: str + ) -> None: + # GIVEN: a runner that never launched anything + runner = self._runner(tmp_path, python_exe) + + # WHEN / THEN: the low-level cancel is a quiet no-op, not an AssertionError + runner._cancel(TerminateCancelMethod()) + + +class TestCancelRacingLaunchIsSerialized: + """The pending-cancel handoff must be atomic with respect to launch. + + `cancel()` runs on another thread. Without serialization there is an + interleaving where the canceller sees a subprocess object and hands off to + `_cancel`, which has nothing to signal because the process has not started + yet, while `_run` has already passed the point where it consumes a pending + cancel -- so the cancel is dropped by both sides and the action runs to + completion. + """ + + @pytest.mark.skipif(not is_posix(), reason="signal delivery is posix-only here") + @pytest.mark.timeout(120) + def test_cancel_during_launch_is_not_dropped(self, tmp_path: Path, python_exe: str) -> None: + # GIVEN: a long-running action, and a canceller that fires while the + # subprocess object exists but has NOT started -- held open by blocking + # inside _start_subprocess, which runs on the pool thread after _run has + # already assigned self._process. + script = StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09("import time; time.sleep(30)"), + ], + ) + ) + ) + runner = StepScriptRunner( + logger=MagicMock(), + session_working_directory=tmp_path, + script=script, + symtab=SymbolTable(), + session_files_directory=tmp_path, + ) + in_window = threading.Event() + canceller_done = threading.Event() + real_start = LoggingSubprocess._start_subprocess + + def _start_after_cancel(subproc): # type: ignore[no-untyped-def] + in_window.set() + # Hold the window: _has_started is not set until this returns. + canceller_done.wait(timeout=30) + return real_start(subproc) + + def _cancel_in_window() -> None: + # try/finally so canceller_done is set no matter what. This runs on its + # own thread, so an exception here would otherwise leave + # _start_after_cancel waiting out its full 30s and the assertions below + # reporting SUCCESS -- a confusing failure a long way from its cause. + try: + in_window.wait(timeout=30) + # Deliberately no assertion on runner._process. This hook runs on + # the pool thread, and `_run` publishes _process and _run_future + # together *after* submitting to the pool, so at this instant + # _process may legitimately still be None. Asserting otherwise made + # this test fail about 1 run in 4 even in isolation: the assertion + # killed its own canceller thread. Both windows -- before _process + # is published, and after it exists but before it has started -- + # must record the cancel rather than drop it, and that is what the + # assertions after the run check. + runner.cancel() + finally: + canceller_done.set() + + canceller = threading.Thread(target=_cancel_in_window, daemon=True) + + # WHEN + with patch.object(LoggingSubprocess, "_start_subprocess", _start_after_cancel): + canceller.start() + runner.run() + canceller.join(timeout=30) + + # THEN: the cancel was applied, so the 30 second sleep did not run to + # completion. Unsynchronized, it is dropped by both sides and the runner + # ends in SUCCESS after 30 seconds. + deadline = time.time() + 60 + while runner.state in (ScriptRunnerState.RUNNING, ScriptRunnerState.CANCELING): + if time.time() > deadline: # pragma: no cover - timing guard + pytest.fail(f"runner never settled; state={runner.state}") + time.sleep(0.05) + assert runner.state == ScriptRunnerState.CANCELED + assert runner._pending_cancel is None diff --git a/test/openjd/sessions_v0/test_session.py b/test/openjd/sessions_v0/test_session.py index f3420a92..b09219d6 100644 --- a/test/openjd/sessions_v0/test_session.py +++ b/test/openjd/sessions_v0/test_session.py @@ -66,6 +66,7 @@ from openjd.sessions._windows_permission_helper import WindowsPermissionHelper from .conftest import ( + serial_process, has_posix_target_user, has_windows_user, WIN_SET_TEST_ENV_VARS_MESSAGE, @@ -954,6 +955,7 @@ def test_run_task_with_variables( ) +@serial_process class TestSessionCancel: """Test that cancelation will cancel the currently running Script.""" @@ -3900,3 +3902,44 @@ def test_run_task_without_session_env_ignores_multiple_environments( # THEN - All entered environment variables should be ignored assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) assert "FOO=NOT_SET" in caplog.messages + + +class TestSessionEmittedLogLinesDoNotTriggerCancel: + """The action-message filter is attached to the session logger, so it also + sees lines the session itself logs while no action is running. + + A task parameter whose name starts with ``openjd_env`` makes ``run_task``'s + own parameter-logging line look like a malformed env macro. The filter then + asks the session to cancel a running action -- and there isn't one, so + ``cancel_action`` used to raise ``RuntimeError`` out of ``logger.info`` and + out of the public API, with no callback and the task never run. + """ + + @pytest.mark.parametrize( + "param_name", + ["openjd_env", "openjd_unset_env", "openjd_redacted_env", "openjd_envX"], + ) + def test_parameter_named_like_an_env_macro_does_not_raise(self, param_name: str) -> None: + # GIVEN + script = StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("ok")], + ) + ) + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + # WHEN: the parameter name collides with the macro prefix + session.run_task( + step_script=script, + task_parameter_values={ + param_name: ParameterValue(type=ParameterValueType.STRING, value="x") + }, + ) + while session.state == SessionState.RUNNING: + time.sleep(0.05) + + # THEN: the task ran normally + assert session.action_status is not None + assert session.action_status.state == ActionState.SUCCESS diff --git a/test/openjd/sessions_v0/test_session_let_bindings.py b/test/openjd/sessions_v0/test_session_let_bindings.py new file mode 100644 index 00000000..47ba8ad1 --- /dev/null +++ b/test/openjd/sessions_v0/test_session_let_bindings.py @@ -0,0 +1,523 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Regression tests for the Session-level handling of EXPR ``let`` +bindings (RFC 0005): + +- a failing ``extra_let_bindings`` entry on ``enter_environment`` / + ``exit_environment`` fails the action through the normal callback path + instead of raising out of the public API; +- ``enter_environment(step_name=...)`` seeds ``Step.Name`` so step-level + bindings and the environment's actions can reference it (on both the + enter and exit sides); +- binding-RHS parsing is memoized across applications; +- the unified optional int-or-format-string field resolver + (``resolve_optional_int_field``) enforces consistent bounds. +""" + +from __future__ import annotations + +import sys +import time +import uuid +from typing import Any + +import pytest + +from openjd.model import SymbolTable +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CommandString as CommandString_2023_09, + Environment as Environment_2023_09, + EnvironmentActions as EnvironmentActions_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, + ModelParsingContext as ModelParsingContext_2023_09, +) +from openjd.model._let_bindings import _parse_rhs +from openjd.sessions import ActionState, ActionStatus, Session, SessionState +from openjd.model.v2023_09 import StepScript as StepScript_2023_09 +from openjd.sessions._runner_base import ( + MAX_INT_FIELD_VALUE, + apply_let_bindings, + resolve_action_arg_values, + resolve_optional_int_field, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _action(command: str, *args: str) -> Action_2023_09: + return Action_2023_09( + command=CommandString_2023_09(command), + args=[ArgString_2023_09(a) for a in args] if args else None, + ) + + +def _env(name: str, **action_kwargs) -> Environment_2023_09: + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09(**action_kwargs), + ), + ) + + +def _run_until_ready(session: Session, timeout_s: float = 10.0) -> None: + deadline = time.time() + timeout_s + while session.state == SessionState.RUNNING and time.time() < deadline: + time.sleep(0.05) + + +def _format_string_field(raw: str) -> Any: + """Build a FormatString-typed ``timeout`` field value (FEATURE_BUNDLE_1).""" + context = ModelParsingContext_2023_09(supported_extensions=["FEATURE_BUNDLE_1", "EXPR"]) + action = Action_2023_09.model_validate({"command": "echo", "timeout": raw}, context=context) + return action.timeout + + +# --------------------------------------------------------------------------- +# A failing extra `let` binding must FAIL the action via the callback path, +# never raise out of enter_environment()/exit_environment(). +# --------------------------------------------------------------------------- + + +class TestExtraLetBindingFailure: + def test_enter_environment_failing_binding_fails_action_cleanly(self) -> None: + # GIVEN: an extra binding referencing an undefined symbol. + callback_events: list[ActionStatus] = [] + + def callback(session_id: str, status: ActionStatus) -> None: + callback_events.append(status) + + env = _env("Env", onEnter=_action("true"), onExit=_action("true")) + with Session( + session_id=uuid.uuid4().hex, job_parameter_values={}, callback=callback + ) as session: + # WHEN: entering must not raise. + identifier = session.enter_environment( + environment=env, + extra_let_bindings=["msg = NoSuchSymbol"], + ) + _run_until_ready(session) + + # THEN: the action failed cleanly, the callback fired, and the + # environment remains entered-but-failed (exactly as a failing + # onEnter subprocess leaves it) so cleanup can exit it. + assert session.state == SessionState.READY_ENDING + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert status.fail_message is not None + assert "let" in status.fail_message + assert callback_events and callback_events[-1].state == ActionState.FAILED + assert identifier in session.environments_entered + + # WHEN: exiting the failed environment re-applies the failing + # bindings — the exit action must also fail cleanly, not raise. + session.exit_environment(identifier=identifier) + _run_until_ready(session) + + # THEN + assert session.state == SessionState.READY_ENDING + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert identifier not in session.environments_entered + + +# --------------------------------------------------------------------------- +# enter_environment(step_name=...) seeds Step.Name (RFC 0005 EXPR), for both +# the enter side and the re-applied bindings on the exit side. +# --------------------------------------------------------------------------- + + +class TestEnterEnvironmentStepName: + def test_step_name_resolvable_in_bindings_and_actions( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # GIVEN: a step-level binding referencing Step.Name, echoed by both + # the environment's onEnter and onExit actions. + env = _env( + "StepEnv", + onEnter=_action("echo", "enter:{{ msg }}"), + onExit=_action("echo", "exit:{{ msg }}"), + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + # WHEN + identifier = session.enter_environment( + environment=env, + extra_let_bindings=["msg = 'step is ' + Step.Name"], + step_name="MyStep", + ) + _run_until_ready(session) + + # THEN: the enter action ran with the binding resolved. + assert session.state == SessionState.READY + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + assert any("enter:step is MyStep" in m for m in caplog.messages) + + # WHEN: the exit re-applies the bindings — Step.Name must be + # re-seeded so onExit resolves in the same scope as onEnter. + session.exit_environment(identifier=identifier) + _run_until_ready(session) + + # THEN + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + assert any("exit:step is MyStep" in m for m in caplog.messages) + + +# --------------------------------------------------------------------------- +# Binding-RHS parsing is memoized: re-applying the same bindings (per task, +# per env enter/exit) must not re-parse through the engine each time. +# --------------------------------------------------------------------------- + + +class TestLetBindingParseMemoization: + def test_parse_is_cached_across_applications(self) -> None: + # GIVEN: apply_let_bindings delegates to the model's single-sourced + # evaluator, whose RHS parse (_parse_rhs) is memoized. + _parse_rhs.cache_clear() + bindings = ["a = 1 + 2", "b = a * 10"] + + # WHEN: the same bindings apply against several symbol tables. + for _ in range(5): + symtab = SymbolTable() + apply_let_bindings(symtab=symtab, let_bindings=bindings) + # THEN: per-application evaluation is unchanged. (Values are the + # engine's typed results; compare via their string rendering.) + assert str(symtab["a"]) == "3" + assert str(symtab["b"]) == "30" + + # THEN: each unique RHS was parsed exactly once. + info = _parse_rhs.cache_info() + assert info.misses == len(bindings) + assert info.hits == 4 * len(bindings) + + def test_cached_expression_evaluates_against_per_call_symtab(self) -> None: + # The memoized expression object must hold no symbol-table state. + _parse_rhs.cache_clear() + first = SymbolTable() + first["X"] = 1 + second = SymbolTable() + second["X"] = 41 + apply_let_bindings(symtab=first, let_bindings=["y = X + 1"]) + apply_let_bindings(symtab=second, let_bindings=["y = X + 1"]) + assert str(first["y"]) == "2" + assert str(second["y"]) == "42" + + +# --------------------------------------------------------------------------- +# A let-bound LIST keeps its engine type through the symbol table and +# flattens through whole-field argument resolution (RFC 0005 §1.3.2) — +# regression for the typed round trip of non-string binding values. +# --------------------------------------------------------------------------- + + +class TestLetBoundListInArgs: + def test_let_bound_list_flattens_into_args(self) -> None: + # GIVEN: a step script whose `let` binds a list and whose onRun + # consumes it as a whole-field argument expression. + context = ModelParsingContext_2023_09(supported_extensions=["EXPR"]) + script = StepScript_2023_09.model_validate( + { + "let": ["files = ['alpha beta', 'gamma']"], + "actions": { + "onRun": { + "command": "echo", + "args": ["front", "{{ files }}", "back"], + } + }, + }, + context=context, + ) + + # WHEN: the bindings are applied the way the runner applies them, + # and the args resolve through the shared enforcement helper. + symtab = SymbolTable() + apply_let_bindings(symtab=symtab, let_bindings=script.let or []) + resolved = resolve_action_arg_values(script.actions.onRun.args, symtab) + + # THEN: the stored value survived the engine symbol-table build as a + # typed list — flattened inline, one argument per element, embedded + # whitespace preserved — not rendered as a single stringified list. + assert resolved == ["front", "alpha beta", "gamma", "back"] + + +# --------------------------------------------------------------------------- +# resolve_optional_int_field: one implementation for the three +# "optional int-or-format-string" call sites, with consistent bounds. +# --------------------------------------------------------------------------- + + +class TestResolveOptionalIntField: + def test_none_field_stays_none(self) -> None: + assert resolve_optional_int_field(None, SymbolTable(), ge=1, description="timeout") is None + + def test_literal_int_passes_through(self) -> None: + # Literal values were bounds-checked by the static validator at + # parse time; they pass through unchecked at run time. + assert resolve_optional_int_field(30, SymbolTable(), ge=1, description="timeout") == 30 + + def test_whole_field_null_resolves_to_none(self) -> None: + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = None + assert resolve_optional_int_field(field, symtab, ge=1, description="timeout") is None + + def test_whole_field_empty_string_rejected(self) -> None: + # A genuine empty STRING is not null (openjd-rs parity: only an + # ExprValue::Null result means "field omitted"; an empty string + # falls through to the integer parse and errors). + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = "" + with pytest.raises(ValueError, match="timeout must be a positive integer, got ''"): + resolve_optional_int_field(field, symtab, ge=1, description="timeout") + + def test_resolved_value_below_ge_rejected(self) -> None: + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = 0 + with pytest.raises(ValueError, match="timeout must be a positive integer, got '0'"): + resolve_optional_int_field(field, symtab, ge=1, description="timeout") + + def test_resolved_value_above_le_rejected(self) -> None: + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = 700 + with pytest.raises( + ValueError, match="notifyPeriodInSeconds must be between 1 and 600, got '700'" + ): + resolve_optional_int_field( + field, symtab, ge=1, le=600, description="notifyPeriodInSeconds" + ) + + @pytest.mark.parametrize( + "value", + [ + pytest.param(" 45 ", id="surrounding whitespace"), + pytest.param("1_0", id="digit-group underscore"), + pytest.param("\u0661\u0662\u0663", id="non-ascii decimal digits"), + ], + ) + def test_lenient_int_spellings_rejected(self, value: str) -> None: + # Strict ASCII integer grammar (openjd-rs parity): Python's int() + # accepts all of these spellings, but Rust's str::parse rejects + # them — accepting them here would be a spec-observable divergence + # for dynamically resolved timeout/notifyPeriodInSeconds values. + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = value + with pytest.raises(ValueError, match="timeout must be a positive integer"): + resolve_optional_int_field(field, symtab, ge=1, description="timeout") + + def test_leading_plus_sign_accepted(self) -> None: + # Rust's u64/i64 from_str accepts a leading '+'; so do we. + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = "+45" + assert resolve_optional_int_field(field, symtab, ge=1, description="timeout") == 45 + + def test_non_integer_resolved_value_rejected(self) -> None: + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = "abc" + with pytest.raises(ValueError, match="timeout must be a positive integer, got 'abc'"): + resolve_optional_int_field(field, symtab, ge=1, description="timeout") + + def test_session_resolve_action_timeout_rejects_non_positive(self) -> None: + # Regression: Session._resolve_action_timeout previously accepted + # any int() — a resolved non-positive timeout must now be rejected, + # matching the openjd-rs runtime and the enforcement path in + # ScriptRunnerBase._run_action. + context = ModelParsingContext_2023_09(supported_extensions=["FEATURE_BUNDLE_1", "EXPR"]) + action = Action_2023_09.model_validate( + {"command": "echo", "timeout": "{{ X }}"}, context=context + ) + symtab = SymbolTable() + symtab["X"] = 0 + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + with pytest.raises(ValueError, match="timeout must be a positive integer"): + session._resolve_action_timeout(action, symtab) + + +# --------------------------------------------------------------------------- +# A runtime EXPR failure in an environment's `variables:` map must FAIL the +# action via the callback path, never raise out of enter_environment(). +# --------------------------------------------------------------------------- + + +class TestEnvironmentVariablesRuntimeFailure: + def test_failing_variable_expression_fails_action_cleanly(self) -> None: + # GIVEN: a `variables:` entry that passes validation but cannot be + # evaluated at run time (int() applied to a non-numeric value). Under + # legacy interpolation this was unreachable; EXPR host functions make it + # reachable for a template that validated successfully. + callback_events: list[ActionStatus] = [] + + def callback(session_id: str, status: ActionStatus) -> None: + callback_events.append(status) + + context = ModelParsingContext_2023_09(supported_extensions=["EXPR"]) + environment = Environment_2023_09.model_validate( + { + "name": "BrokenVars", + "variables": {"BROKEN": "{{ int(Session.WorkingDirectory.name) }}"}, + "script": {"actions": {"onEnter": {"command": "echo", "args": ["entered"]}}}, + }, + context=context, + ) + + with Session( + session_id=uuid.uuid4().hex, job_parameter_values={}, callback=callback + ) as session: + # WHEN: entering must not raise. + identifier = session.enter_environment(environment=environment) + _run_until_ready(session) + + # THEN: the caller gets the identifier, a terminal FAILED status, and + # a session it can still exit the attempted environment from. + assert identifier is not None + assert session.state == SessionState.READY_ENDING + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert status.fail_message is not None + assert "environment variables" in status.fail_message + assert callback_events and callback_events[-1].state == ActionState.FAILED + assert identifier in session.environments_entered + + # AND: the change record was seeded, so the log-forwarding thread + # cannot KeyError on an openjd_env emitted by this environment. + assert identifier in session._created_env_vars + + # WHEN: cleanup exits it + session.exit_environment(identifier=identifier) + _run_until_ready(session) + + # THEN + assert identifier not in session.environments_entered + + +# --------------------------------------------------------------------------- +# RFC 0005 1.3.2 typed argument semantics on the ENFORCEMENT path: what a real +# subprocess receives, not just what the helper returns. A whole-field list +# expression flattens to one argument per element; a whole-field null is +# skipped entirely. +# --------------------------------------------------------------------------- + + +class TestTypedArgumentSemanticsEndToEnd: + def test_subprocess_argv_flattens_lists_and_skips_nulls( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # GIVEN: a step whose args mix plain strings, a whole-field list + # expression, and a whole-field null. + context = ModelParsingContext_2023_09(supported_extensions=["EXPR"]) + step = StepScript_2023_09.model_validate( + { + "let": ["items = ['alpha beta', 'gamma']", "missing = null"], + "actions": { + "onRun": { + "command": sys.executable, + "args": [ + "-c", + "import sys; print('ARGV=' + repr(sys.argv[1:]))", + "front", + "{{items}}", + "{{missing}}", + "back", + ], + } + }, + }, + context=context, + ) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + # WHEN + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + # THEN: four arguments -- the list flattened element-wise (preserving the + # space inside an element) and the null argument was dropped. A stringified + # list or an empty argument would both be wrong. + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + expected = "ARGV=" + repr(["front", "alpha beta", "gamma", "back"]) + assert any(expected in m for m in caplog.messages), [ + m for m in caplog.messages if "ARGV=" in m + ] + + +# --------------------------------------------------------------------------- +# The over-range bound applies to a value that arrives through a format string, +# not only to a literal -- that is the path a forwarded WrappedAction.Timeout +# takes, and without the bound the action would run unbounded instead of failing. +# --------------------------------------------------------------------------- + + +class TestOverRangeResolvedIntField: + def test_resolved_value_above_max_is_rejected(self) -> None: + # GIVEN + symtab = SymbolTable() + symtab["X"] = str(MAX_INT_FIELD_VALUE + 1) + + # WHEN / THEN + with pytest.raises(ValueError, match="must be at most"): + resolve_optional_int_field( + _format_string_field("{{ X }}"), symtab, ge=1, description="timeout" + ) + + def test_resolved_value_at_max_is_accepted(self) -> None: + # GIVEN + symtab = SymbolTable() + symtab["X"] = str(MAX_INT_FIELD_VALUE) + + # WHEN / THEN: the boundary itself is pinned from both sides. + assert ( + resolve_optional_int_field( + _format_string_field("{{ X }}"), symtab, ge=1, description="timeout" + ) + == MAX_INT_FIELD_VALUE + ) + + +# --------------------------------------------------------------------------- +# Task.File.* / Env.File.* are PATH-typed, so EXPR property access works on +# them. Without the type they resolve as plain strings and `.parent` fails. +# --------------------------------------------------------------------------- + + +class TestEmbeddedFileSymbolsArePathTyped: + def test_task_file_supports_path_property_access( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # GIVEN: a step-level binding taking `.parent` of an embedded file path. + context = ModelParsingContext_2023_09(supported_extensions=["EXPR"]) + step = StepScript_2023_09.model_validate( + { + "let": ["where = string(Task.File.Cfg.parent)"], + "actions": {"onRun": {"command": "echo", "args": ["PARENT={{where}}"]}}, + "embeddedFiles": [{"name": "Cfg", "type": "TEXT", "data": "config-contents\n"}], + }, + context=context, + ) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + # WHEN + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + # THEN: the binding resolved, so the symbol carried the PATH type. + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + assert any("PARENT=" in m and "embedded_files" in m for m in caplog.messages) diff --git a/test/openjd/sessions_v0/test_session_run_subprocess.py b/test/openjd/sessions_v0/test_session_run_subprocess.py index 809c86e7..6c464cb5 100644 --- a/test/openjd/sessions_v0/test_session_run_subprocess.py +++ b/test/openjd/sessions_v0/test_session_run_subprocess.py @@ -21,6 +21,7 @@ from openjd.sessions._session_user import PosixSessionUser, WindowsSessionUser from .conftest import ( + serial_process, has_posix_target_user, has_windows_user, WIN_SET_TEST_ENV_VARS_MESSAGE, @@ -28,6 +29,7 @@ ) +@serial_process class TestRunSubprocess: """Tests for the Session.run_subprocess method.""" diff --git a/test/openjd/sessions_v0/test_subprocess.py b/test/openjd/sessions_v0/test_subprocess.py index 6c014546..27b86e96 100644 --- a/test/openjd/sessions_v0/test_subprocess.py +++ b/test/openjd/sessions_v0/test_subprocess.py @@ -18,8 +18,10 @@ from openjd.sessions._os_checker import is_posix, is_windows from openjd.sessions._session_user import PosixSessionUser, WindowsSessionUser from openjd.sessions._subprocess import LoggingSubprocess +from openjd.sessions import _subprocess as subprocess_impl_mod from .conftest import ( + serial_process, build_logger, collect_queue_messages, has_posix_target_user, @@ -229,6 +231,7 @@ def test_invokes_callback(self, queue_handler: QueueHandler, python_exe: str) -> # THEN callback_mock.assert_called_once() + @serial_process def test_notify_ends_process( self, message_queue: SimpleQueue, queue_handler: QueueHandler, python_exe: str ) -> None: @@ -270,6 +273,7 @@ def end_proc(): assert "Log from test 9" not in all_messages assert subproc.exit_code != 0 + @serial_process def test_terminate_ends_process( self, message_queue: SimpleQueue, queue_handler: QueueHandler, python_exe: str ) -> None: @@ -316,6 +320,7 @@ def end_proc(): os.environ.get("CODEBUILD_BUILD_ID", None) is not None, reason="This test is failing exclusively in codebuild; unblocking, and will root cause later.", ) + @serial_process def test_terminate_ends_process_tree( self, message_queue: SimpleQueue, @@ -1069,3 +1074,61 @@ def end_proc(): if num_children_running == 0: break assert num_children_running == 0 + + +class TestFastExitingChild: + """A trivial command can exit before the runner finishes recording it. + + Looking up an already-reaped child's process group raises ProcessLookupError + on posix, and psutil raises NoSuchProcess when walking it on Windows. Neither + may fail the action: the child ran, and its exit code is still collected. + """ + + @pytest.mark.skipif(not is_posix(), reason="posix-only: process groups") + @pytest.mark.usefixtures("message_queue", "queue_handler") + def test_getpgid_lookup_failure_does_not_fail_the_action( + self, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + # GIVEN: the child is gone by the time its process group is looked up + logger = build_logger(queue_handler) + callback = MagicMock() + subproc = LoggingSubprocess( + logger=logger, + args=[sys.executable, "-c", "print('DONE')"], + callback=callback, + ) + + with patch.object( + subprocess_impl_mod.os, "getpgid", side_effect=ProcessLookupError(3, "No such process") + ): + # WHEN + subproc.run() + + # THEN: the action completed normally + assert subproc.exit_code == 0 + assert subproc.failed_to_start is False + messages = collect_queue_messages(message_queue) + assert "DONE" in messages + + @pytest.mark.skipif(not is_windows(), reason="Windows-only: psutil process walk") + def test_process_tree_walk_tolerates_exited_process(self) -> None: + # GIVEN: a process that disappears between discovery and the walk + from psutil import NoSuchProcess + + from openjd.sessions._windows_process_killer import _suspend_process_tree + + logger = MagicMock() + process = MagicMock() + process.pid = 4321 + process.suspend.side_effect = NoSuchProcess(4321) + process.children.side_effect = NoSuchProcess(4321) + cannot_suspend: list = [] + all_processes: list = [] + + # WHEN / THEN: no exception escapes + _suspend_process_tree( + logger, process, all_processes, cannot_suspend, suspend_subprocesses=True + ) + assert process in all_processes diff --git a/test/openjd/sessions_v0/test_subprocess_process_group.py b/test/openjd/sessions_v0/test_subprocess_process_group.py new file mode 100644 index 00000000..b088be76 --- /dev/null +++ b/test/openjd/sessions_v0/test_subprocess_process_group.py @@ -0,0 +1,89 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Recording the process group of a launched subprocess. + +The recorded group is the only signal target a later cancel has, so an unknown +group has to be recorded as unknown rather than guessed. +""" + +import sys +from logging.handlers import QueueHandler +from queue import SimpleQueue +from unittest.mock import MagicMock, patch + +import pytest + + +from openjd.sessions._os_checker import is_posix +from openjd.sessions._subprocess import LoggingSubprocess + +from .conftest import build_logger + + +@pytest.mark.skipif(not is_posix(), reason="process groups are POSIX-only") +class TestProcessGroupIsUnknownWhenItCannotBeLookedUp: + """R5-9: the lookup fails *because* the process is gone, so its pid is dead. + Recording it as a process-group id means a later `killpg` is at best a no-op + and, after pid recycling, targets an unrelated group.""" + + def test_reaped_child_leaves_the_group_unknown( + self, message_queue: SimpleQueue, queue_handler: QueueHandler + ) -> None: + # GIVEN: a child whose process group cannot be looked up + logger = build_logger(queue_handler) + proc = LoggingSubprocess( + logger=logger, args=[sys.executable, "-c", "pass"], os_env_vars=None + ) + + # WHEN + with patch( + "openjd.sessions._subprocess.os.getpgid", side_effect=ProcessLookupError(3, "No such") + ): + proc.run() + + # THEN: "unknown", not a stale pid -- and the action still succeeded, + # which is the behaviour the original fix existed to protect. + assert proc._sudo_child_process_group_id is None + assert proc.exit_code == 0 + assert proc.failed_to_start is False + + def test_no_signal_is_delivered_when_the_group_is_unknown( + self, message_queue: SimpleQueue, queue_handler: QueueHandler + ) -> None: + # GIVEN: a finished process with no known group + logger = build_logger(queue_handler) + proc = LoggingSubprocess(logger=logger, args=[sys.executable, "-c", "pass"]) + with patch( + "openjd.sessions._subprocess.os.getpgid", side_effect=ProcessLookupError(3, "No such") + ): + proc.run() + assert proc._sudo_child_process_group_id is None + + # WHEN: a SIGKILL is attempted anyway + with ( + patch("openjd.sessions._subprocess.os.killpg") as killpg, + patch( + "openjd.sessions._subprocess.find_sudo_child_process_group_id", return_value=None + ), + ): + proc._posix_signal_subprocess(MagicMock(pid=999999), signal_name="kill") + + # THEN: nothing was signalled. + killpg.assert_not_called() + + def test_sudo_helper_returns_unknown_when_sudo_is_already_gone(self) -> None: + """Sibling: the same guard on the first getpgid in the sudo helper.""" + # GIVEN + from openjd.sessions._linux._sudo import find_sudo_child_process_group_id + + # WHEN + with patch( + "openjd.sessions._linux._sudo.os.getpgid", + side_effect=ProcessLookupError(3, "No such process"), + ): + result = find_sudo_child_process_group_id( + logger=MagicMock(), sudo_process=MagicMock(pid=999999) + ) + + # THEN: the established "unknown" value, not an escaping ESRCH. + assert result is None diff --git a/test/openjd/sessions_v0/test_subprocess_reaping.py b/test/openjd/sessions_v0/test_subprocess_reaping.py new file mode 100644 index 00000000..703fb38d --- /dev/null +++ b/test/openjd/sessions_v0/test_subprocess_reaping.py @@ -0,0 +1,340 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""``LoggingSubprocess.run()`` owns its child on every exit path. + +``run()`` is the only place that holds the ``Popen`` handle, and clearing +``self._process`` in its ``finally`` is the point after which nothing can reach +the child again -- ``notify()`` and ``terminate()`` both become permanent no-ops. +So ownership has to be discharged before that, on every path, not only on the one +that reaches ``wait()``. + +It was not. ``_log_subproc_stdout()``, ``wait()`` and the returncode capture all +sat in one ``try``, so any exception out of the stdout pump skipped the wait and +the capture while still clearing the handle. The result was a live child with no +owner, no exit code, and an object reporting neither ``is_running`` nor +``failed_to_start``. + +The trigger is a ``logging.Filter`` that raises. Filters run inside +``Logger.handle``, so they propagate into the pump, and installing one is a +supported use of this library -- ``Session`` installs its own +(``ActionMonitoringFilter``). Hardening that one filter, as an earlier change did, +does not close the structural hole here. +""" + +from __future__ import annotations + +import logging +import os +import signal +import time +from logging.handlers import QueueHandler +from queue import SimpleQueue +from typing import Any, Optional +from unittest.mock import MagicMock, patch + +import pytest + +from openjd.sessions._os_checker import is_posix +from openjd.sessions._subprocess import LoggingSubprocess + +from .conftest import build_logger + +# A child that emits one line (to drive the pump), then outlives the pump. +_LONG_CHILD = "import time\nprint('MARKER', flush=True)\ntime.sleep(60)\n" +# A child that emits one line and exits promptly with a known code. +_SHORT_CHILD = "import sys\nprint('MARKER', flush=True)\nsys.exit(7)\n" + + +class _ExplodingFilter(logging.Filter): + """Raises when the child's output passes through it.""" + + def __init__(self, marker: str = "MARKER") -> None: + super().__init__() + self._marker = marker + self.fired = False + + def filter(self, record: logging.LogRecord) -> bool: + if self._marker in str(record.msg): + self.fired = True + raise RuntimeError("a log filter blew up while forwarding child output") + return True + + +def _pid_alive(pid: int) -> bool: + """Is `pid` a process we can still signal? + + On POSIX a reaped child is gone from the table, so ``kill(pid, 0)`` raising + ESRCH is the reap check. A *zombie* would still answer signal 0, so this + distinguishes "terminated and reaped" from "terminated and leaked". + """ + try: + os.kill(pid, 0) + except OSError: + return False + return True + + +def _wait_gone(pid: Optional[int], timeout_s: float = 15.0) -> bool: + if pid is None: + return True + deadline = time.time() + timeout_s + while time.time() < deadline: + if not _pid_alive(pid): + return True + time.sleep(0.05) + return False + + +@pytest.fixture +def exploding_logger() -> Any: + """A logger whose filter raises on the child's output, plus the filter.""" + exploder = _ExplodingFilter() + logger = logging.getLogger(f"openjd.test.reaping.{id(exploder)}") + logger.setLevel(logging.INFO) + logger.addHandler(logging.NullHandler()) + logger.addFilter(exploder) + try: + yield logger, exploder + finally: + logger.removeFilter(exploder) + + +@pytest.mark.skipif(not is_posix(), reason="pid liveness/reap checks are POSIX-only here") +class TestPumpExceptionDoesNotAbandonTheChild: + def test_live_child_is_terminated_and_reaped( + self, exploding_logger: Any, python_exe: str + ) -> None: + # GIVEN: a long-running child and a log filter that will raise on its + # first line of output, while the child is still alive + logger, exploder = exploding_logger + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _LONG_CHILD]) + + # WHEN + with pytest.raises(RuntimeError, match="blew up"): + proc.run() + + # THEN: the filter really did fire mid-pump (otherwise this test proves + # nothing about the abandonment path)... + assert exploder.fired is True + pid = proc.pid + assert pid is not None + # ...and the child is gone, not merely signalled. + assert _wait_gone(pid), f"child {pid} was left running" + + def test_exit_code_is_recorded_for_an_abandoned_child( + self, exploding_logger: Any, python_exe: str + ) -> None: + """The exit code is the only evidence of what happened to the action, and + it used to be lost entirely.""" + # GIVEN + logger, _ = exploding_logger + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _LONG_CHILD]) + + # WHEN + with pytest.raises(RuntimeError): + proc.run() + + # THEN: a real exit status, not None. SIGKILL shows as -SIGKILL. + assert proc.exit_code is not None + assert proc.exit_code == -signal.SIGKILL # type: ignore + + def test_a_child_that_exited_on_its_own_keeps_its_own_exit_code( + self, exploding_logger: Any, python_exe: str + ) -> None: + """The reap must not overwrite a genuine exit status with a kill signal.""" + # GIVEN: a child that exits 7 by itself, and a filter that raises + logger, _ = exploding_logger + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _SHORT_CHILD]) + + # WHEN + with pytest.raises(RuntimeError): + proc.run() + + # THEN: whichever way the race went, the recorded code is a real one and + # never None. If the child had already exited, it must be its own 7. + assert proc.exit_code is not None + assert proc.exit_code in (7, -signal.SIGKILL) # type: ignore + + def test_a_reap_failure_does_not_replace_the_original_exception( + self, exploding_logger: Any, python_exe: str + ) -> None: + """The reap runs inside `run()`'s `finally`, so anything it raises would + replace the exception already propagating -- hiding the real cause. + + Goes through `run()` rather than calling `_reap` directly: the + containment being pinned is the `finally`'s, and a test that calls + `_reap` itself never enters that `finally` at all. An earlier version of + this file made that mistake, and narrowing the reap's `except Exception` + to `except OSError` passed every test. + + The injected failure is deliberately NOT an OSError, so a handler that + only catches OSError lets it through. + """ + # GIVEN: a live child, a filter that raises, and a termination that fails + # with a non-OSError + logger, exploder = exploding_logger + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _LONG_CHILD]) + + # WHEN + with ( + patch.object( + LoggingSubprocess, + "_terminate_process", + side_effect=RuntimeError("terminate itself blew up"), + ), + patch("openjd.sessions._subprocess.ABANDONED_PROCESS_REAP_TIMEOUT_SECONDS", 0), + ): + with pytest.raises(RuntimeError) as excinfo: + proc.run() + + # THEN: the caller sees the pump's failure, which is the real cause -- not + # the reap's. + assert exploder.fired is True + assert "blew up while forwarding child output" in str(excinfo.value) + assert "terminate itself blew up" not in str(excinfo.value) + # ...and the handle is still released. + assert proc._process is None + + # Clean up the child the failed terminate left behind. + if proc.pid is not None and _pid_alive(proc.pid): + os.kill(proc.pid, signal.SIGKILL) # type: ignore + try: + os.waitpid(proc.pid, 0) + except ChildProcessError: + # Already reaped -- which is the outcome this test wants anyway. + # Popen's own destructor or the code under test may have got there + # first; either way there is no zombie left to collect. + pass + + def test_the_process_handle_is_still_released( + self, exploding_logger: Any, python_exe: str + ) -> None: + """Reaping must not come at the cost of the deallocation the `finally` + existed for.""" + # GIVEN / WHEN + logger, _ = exploding_logger + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _LONG_CHILD]) + with pytest.raises(RuntimeError): + proc.run() + + # THEN + assert proc._process is None + assert proc.is_running is False + + +@pytest.mark.skipif(not is_posix(), reason="POSIX signal semantics") +class TestNormalPathIsUnchanged: + """The reap is a no-op when `run()` completed properly. These are the tests + that would catch it terminating a healthy child or clobbering its status.""" + + def test_successful_run_keeps_its_exit_code_and_is_not_signalled( + self, message_queue: SimpleQueue, queue_handler: QueueHandler, python_exe: str + ) -> None: + # GIVEN + logger = build_logger(queue_handler) + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _SHORT_CHILD]) + + # WHEN + proc.run() + + # THEN: its own exit code, positive -- not a negative signal value. + assert proc.exit_code == 7 + assert proc.failed_to_start is False + + def test_successful_run_does_not_terminate_the_child( + self, message_queue: SimpleQueue, queue_handler: QueueHandler, python_exe: str + ) -> None: + """Directly pins that the new path does not fire on a healthy run.""" + # GIVEN + logger = build_logger(queue_handler) + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", "pass"]) + + # WHEN + with patch.object(LoggingSubprocess, "_terminate_process", autospec=True) as terminate: + proc.run() + + # THEN + terminate.assert_not_called() + assert proc.exit_code == 0 + + def test_output_is_still_forwarded( + self, message_queue: SimpleQueue, queue_handler: QueueHandler, python_exe: str + ) -> None: + # GIVEN / WHEN + logger = build_logger(queue_handler) + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _SHORT_CHILD]) + proc.run() + + # THEN + lines = [] + while not message_queue.empty(): + lines.append(message_queue.get().getMessage()) + assert "MARKER" in lines + + +@pytest.mark.skipif(not is_posix(), reason="POSIX signal semantics") +class TestReapUnitBehaviour: + """Direct coverage of `_reap` for the branches the end-to-end tests cannot + reach cheaply.""" + + def test_reap_of_none_is_a_no_op(self) -> None: + # GIVEN a subprocess that never started + proc = LoggingSubprocess(logger=MagicMock(), args=["/bin/echo", "hi"]) + + # WHEN / THEN: no exception, and no exit code invented + proc._reap(None) + assert proc.exit_code is None + + def test_a_terminate_failure_does_not_escape(self, python_exe: str) -> None: + """`_reap` runs inside a `finally`. An exception from it would replace + whatever exception was already propagating.""" + # GIVEN a live child whose termination will fail + logger = MagicMock() + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _LONG_CHILD]) + popen = proc._start_subprocess() + assert popen is not None + try: + with ( + patch.object( + LoggingSubprocess, + "_terminate_process", + side_effect=OSError("cannot signal"), + ), + patch( + # The child survives, since termination was made to fail, so the + # wait that follows must not burn its real budget here. + "openjd.sessions._subprocess.ABANDONED_PROCESS_REAP_TIMEOUT_SECONDS", + 0, + ), + ): + # WHEN / THEN: swallowed and logged, not raised + proc._reap(popen) + assert any("Could not terminate" in str(call) for call in logger.error.call_args_list) + finally: + popen.kill() + popen.wait() + + def test_a_child_that_survives_termination_is_reported_not_hung(self, python_exe: str) -> None: + """The wait is bounded: a child that somehow outlives SIGKILL must not + hang the pool worker.""" + # GIVEN a live child, a termination that does nothing, and a 0s budget + logger = MagicMock() + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _LONG_CHILD]) + popen = proc._start_subprocess() + assert popen is not None + try: + with ( + patch.object(LoggingSubprocess, "_terminate_process"), + patch("openjd.sessions._subprocess.ABANDONED_PROCESS_REAP_TIMEOUT_SECONDS", 0), + ): + start = time.monotonic() + # WHEN + proc._reap(popen) + elapsed = time.monotonic() - start + + # THEN: returned promptly, and said so. + assert elapsed < 10.0 + assert any("did not exit within" in str(call) for call in logger.error.call_args_list) + finally: + popen.kill() + popen.wait() diff --git a/test/openjd/sessions_v0/test_tempdir.py b/test/openjd/sessions_v0/test_tempdir.py index 1aa01c83..77f5c122 100644 --- a/test/openjd/sessions_v0/test_tempdir.py +++ b/test/openjd/sessions_v0/test_tempdir.py @@ -1,11 +1,12 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -from tempfile import gettempdir +from tempfile import gettempdir, mkdtemp import os import shutil import stat from pathlib import Path from subprocess import DEVNULL, run +from typing import Any, Callable from openjd.sessions._os_checker import is_posix, is_windows from openjd.sessions._windows_permission_helper import WindowsPermissionHelper @@ -33,14 +34,181 @@ has_posix_disjoint_user, has_posix_target_user, has_windows_user, + nonexistent_group_name, + resolvable_member_groups, WIN_SET_TEST_ENV_VARS_MESSAGE, POSIX_SET_TARGET_USER_ENV_VARS_MESSAGE, POSIX_SET_DISJOINT_USER_ENV_VARS_MESSAGE, ) +def spy_mkdtemp(created: list[str]) -> Callable[..., str]: + """A `mkdtemp` that records what it creates, for asserting on a directory whose + path the failed constructor never handed back. + + Recording the exact path, rather than diffing a directory listing, is what + makes the assertion safe where the session root is shared: another test + creating its own session directory there cannot affect it. + """ + + def _mkdtemp(**kwargs: Any) -> str: + path = mkdtemp(**kwargs) + created.append(path) + return path + + return _mkdtemp + + @pytest.mark.skipif(not is_posix(), reason="Posix-specific tests") class TestTempDirPosix: + def test_unresolvable_group_raises_runtimeerror(self, tmp_path: Path) -> None: + """Pins: a group name that does not resolve must fail TempDir() as + RuntimeError, the error this constructor documents. + + shutil.chown raises LookupError for an unknown group, and LookupError is + neither OSError nor ValueError, so it escaped the `except OSError` around + the ownership change here (and every other handler in the session-setup + chain) and reached the caller of the public Session API unchanged. + PosixSessionUser does not validate its group, so a caller only has to + pass a group that does not exist. + + pytest.raises(RuntimeError) does not catch LookupError, so if the + translation is removed this test errors out with the escaping + LookupError -- which is the defect itself. + """ + # GIVEN + # Only `group` matters on this path; the user is never resolved by it. + user = PosixSessionUser(user="nobody", group=nonexistent_group_name()) + + # WHEN + with pytest.raises(RuntimeError) as excinfo: + TempDir(dir=tmp_path, user=user) + + # THEN + # The failure is the group ownership change, not something incidental: + # the offending group name is carried through to the message. + assert user.group in str(excinfo.value) + + def test_failed_ownership_change_leaves_no_directory(self, tmp_path: Path) -> None: + """Pins: a construction that fails at the ownership step removes the + directory it had already created. + + mkdtemp() runs before the ownership change, so a failure there used to + leave the directory behind with no way to reach it: __init__ raised, so + the caller never received a TempDir and has no path to call cleanup() + with. In production that directory is not under a per-test tmp_path but + under the shared `/OpenJD` root, which nothing else ever prunes, + so each failed session leaks one more 0o700 directory there. + + Asserting on the directory listing, rather than on the exception alone, is + what makes this a pin: the pre-fix code raises the same RuntimeError. + """ + # GIVEN + # An unresolvable group reaches the failure path without needing any + # privileges to arrange: PosixSessionUser does not validate its group. + user = PosixSessionUser(user="nobody", group=nonexistent_group_name()) + before = set(os.listdir(tmp_path)) + created: list[str] = [] + + # WHEN + # Matched, not bare: this constructor raises RuntimeError for several + # earlier failures too (mkdtemp itself, for one), and those never create a + # directory, so a bare match could pass without the leak path running. + with patch("openjd.sessions._tempdir.mkdtemp", side_effect=spy_mkdtemp(created)): + with pytest.raises(RuntimeError, match="Could not change ownership"): + TempDir(dir=tmp_path, user=user) + + # THEN + assert set(os.listdir(tmp_path)) == before + # AND, named exactly rather than inferred from the listing: TempDir resolves + # `dir` before creating anything, so a listing on its own could in principle + # be looking at the wrong directory. + assert created, "test setup: mkdtemp was not reached, so nothing could have leaked" + assert not os.path.exists(created[0]) + + def test_successful_ownership_change_keeps_the_directory(self, tmp_path: Path) -> None: + """The counterpart: the cleanup-on-failure must not fire on success. + + A guard that removed the directory unconditionally would pass the leak + test above and break every real session, so the success path is pinned in + the same place -- including the widened mode, since that is set after the + ownership change and inside the same guarded block. + """ + # GIVEN a group this process is a member of whose gid the new directory + # would NOT already have. chown to one's own group needs no privileges, so + # this exercises the real ownership path on any POSIX host without a + # provisioned test user -- but only a *different* gid proves the chown + # happened, since mkdtemp already inherits the parent's gid. + inherited_gid = os.stat(tmp_path).st_gid + candidates = [ + (gid, name) for gid, name in resolvable_member_groups() if gid != inherited_gid + ] + if not candidates: + pytest.skip("this process is a member of no group other than the one it would inherit") + gid, group_name = candidates[0] + user = PosixSessionUser(user=pwd.getpwuid(os.geteuid()).pw_name, group=group_name) # type: ignore + + # WHEN + result = TempDir(dir=tmp_path, user=user) + + # THEN + assert result.path.is_dir() + statinfo = os.stat(result.path) + assert statinfo.st_gid == gid + # 0o770: mkdtemp's 0o700 widened for the group, and no wider. + assert stat.S_IMODE(statinfo.st_mode) == stat.S_IRWXU | stat.S_IRWXG + + @pytest.mark.skipif( + os.geteuid() == 0 if is_posix() else True, # type: ignore + reason="root is not subject to directory permissions", + ) + def test_cleanup_failure_does_not_replace_the_ownership_error(self, tmp_path: Path) -> None: + """Pins: the cleanup is best-effort, and the failure it cleans up after is + still the one the caller sees. + + An exception raised inside an `except` block replaces the one being + handled. If the removal were not best-effort, a session whose group could + not be set would report a permission error about a temporary directory + instead of naming the group -- turning a diagnosable configuration + mistake into a confusing one, and only on the hosts where cleanup + happens to fail. + + The removal is made to fail for real, by taking write permission off the + parent directory at the moment the ownership change fails: rmdir() needs + write access to the parent, not to the directory being removed. + + Skipped as root, which ignores directory permissions -- so a green run as + root is not evidence for this behaviour. It is covered by the + `localuser_sudo_environment` container, which runs the suite as `hostuser`. + """ + # GIVEN + user = PosixSessionUser(user="nobody", group=nonexistent_group_name()) + before = set(os.listdir(tmp_path)) + original_mode = stat.S_IMODE(os.stat(tmp_path).st_mode) + + def fail_and_make_the_parent_unwritable(path: Path, group: str) -> None: + os.chmod(tmp_path, stat.S_IRUSR | stat.S_IXUSR) + raise OSError(f"no such group: {group}") + + try: + with patch( + "openjd.sessions._tempdir.chown_group", + side_effect=fail_and_make_the_parent_unwritable, + ): + # WHEN + with pytest.raises(RuntimeError) as excinfo: + TempDir(dir=tmp_path, user=user) + + # THEN the ownership failure survived the failed cleanup. + assert user.group in str(excinfo.value) + # AND the cleanup really did fail, so this test is exercising the + # best-effort path rather than the ordinary one. + assert set(os.listdir(tmp_path)) != before + finally: + # Restored to what it was, so that this test's temporary directory can + # be collected however pytest chose to create it. + os.chmod(tmp_path, original_mode) + def test_defaults(self) -> None: # GIVEN tmpdir = Path(os.path.join(gettempdir(), "OpenJD")).resolve() @@ -173,10 +341,20 @@ def test_windows_permissions_inherited(self, mock_user_match, windows_user: Wind def test_nonvalid_windows_principal_raises_exception(self, mock_user_match): # GIVEN windows_user = WindowsSessionUser("non_existent_user") + created: list[str] = [] # THEN - with pytest.raises(RuntimeError, match="Could not change permissions of directory"): - TempDir(user=windows_user) + with patch("openjd.sessions._tempdir.mkdtemp", side_effect=spy_mkdtemp(created)): + with pytest.raises(RuntimeError, match="Could not change permissions of directory"): + TempDir(user=windows_user) + + # AND the directory that had already been created was removed. This is the + # Windows half of the same leak as + # TestTempDirPosix::test_failed_ownership_change_leaves_no_directory, and it + # matters more here: no `dir` was passed, so the directory was created in + # the shared `%PROGRAMDATA%\Amazon\OpenJD` root that nothing else prunes. + assert created, "test setup: mkdtemp was not reached, so nothing could have leaked" + assert not os.path.exists(created[0]) @pytest.fixture def clean_up_directory(self): diff --git a/test/openjd/sessions_v0/test_tempdir_hardening.py b/test/openjd/sessions_v0/test_tempdir_hardening.py new file mode 100644 index 00000000..950ab5f0 --- /dev/null +++ b/test/openjd/sessions_v0/test_tempdir_hardening.py @@ -0,0 +1,455 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Hardening of the shared session temporary root and its cleanup. + +``/OpenJD`` is a fixed, predictable path whose parent is world-writable +on typical POSIX hosts, and it is shared with the job user. +""" + +import os +import stat +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Iterator +from unittest.mock import patch + +import pytest + +from openjd.sessions._os_checker import is_posix, is_windows +from openjd.sessions._tempdir import ( + OPENJD_TEMPDIR_MODE, + TempDir, + _prepare_temp_dir_root, + _prepare_temp_dir_root_windows, + custom_gettempdir, +) + + +def temp_root_under(parent: Path) -> Path: + """The root `custom_gettempdir()` will use when its parent is redirected to `parent`. + + The two platforms nest differently: POSIX uses `/OpenJD`, Windows + uses `%PROGRAMDATA%\\Amazon\\OpenJD`. + """ + return parent / "Amazon" / "OpenJD" if is_windows() else parent / "OpenJD" + + +@contextmanager +def redirected_temp_root(parent: Path) -> Iterator[Path]: + """Redirect `custom_gettempdir()` beneath `parent` on either platform. + + Patching `gettempdir` alone is not enough: on Windows `custom_gettempdir()` + never calls it, so a `gettempdir`-only patch silently leaves the test + operating on the real `%PROGRAMDATA%\\Amazon\\OpenJD`. + """ + if is_windows(): + with patch.dict(os.environ, {"PROGRAMDATA": str(parent)}): + yield temp_root_under(parent) + else: + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + yield temp_root_under(parent) + + +class TestSharedTempRootValidation: + """R5-3: `/OpenJD` is a fixed, predictable path whose parent is + world-writable on typical POSIX hosts. `exist_ok=True` accepted whatever was + already there.""" + + def test_created_with_an_explicit_mode_regardless_of_umask(self, tmp_path: Path) -> None: + # GIVEN: a hostile umask that would otherwise strip group/other bits + old_umask = os.umask(0o077) + try: + with redirected_temp_root(tmp_path) as expected_root: + # WHEN + created = custom_gettempdir() + finally: + os.umask(old_umask) + + # THEN: the root is traversable as intended, not umask-dependent. + assert Path(created) == expected_root + if is_posix(): + assert stat.S_IMODE(os.stat(created).st_mode) == stat.S_IMODE(OPENJD_TEMPDIR_MODE) + + @pytest.mark.skipif(not is_posix(), reason="symlink pre-creation is a POSIX vector here") + def test_rejects_a_symlink_at_the_root_path(self, tmp_path: Path) -> None: + # GIVEN: an attacker has replaced the root with a symlink elsewhere + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + parent = tmp_path / "parent" + parent.mkdir() + (parent / "OpenJD").symlink_to(elsewhere, target_is_directory=True) + + # WHEN / THEN: we refuse rather than creating sessions inside it. The + # refusal now comes from O_NOFOLLOW on the open itself (see REG-3). + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with pytest.raises(RuntimeError, match="real directory"): + custom_gettempdir() + + def test_rejects_a_root_that_is_not_a_directory(self, tmp_path: Path) -> None: + # GIVEN: a plain file squatting on the root path + parent = tmp_path / "parent" + squatter = temp_root_under(parent) + squatter.parent.mkdir(parents=True) + squatter.write_text("squat") + + # WHEN / THEN: makedirs() rejects it first on both platforms, so this + # asserts the refusal, not which layer produced it. + with redirected_temp_root(parent): + with pytest.raises(RuntimeError): + custom_gettempdir() + + @staticmethod + def _fstat_reporting_uid(uid: int) -> Any: + """Wrap os.fstat so the root directory appears owned by `uid`. + + Patches fstat rather than lstat: the implementation validates through a + descriptor now (see REG-3), so an lstat mock would not be consulted and + the test would vacuously pass. + """ + real_fstat = os.fstat + + class _Stat: + def __init__(self, base: os.stat_result) -> None: + self.st_mode = base.st_mode + self.st_uid = uid + + def fake_fstat(fd: int, *a: Any, **k: Any) -> Any: + return _Stat(real_fstat(fd, *a, **k)) + + return fake_fstat + + @pytest.mark.skipif(not is_posix(), reason="uid ownership check is POSIX-only") + def test_rejects_a_root_owned_by_another_user(self, tmp_path: Path) -> None: + # GIVEN: the root exists but is owned by a different uid + parent = tmp_path / "parent" + parent.mkdir() + (parent / "OpenJD").mkdir() + + # WHEN / THEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with patch( + "openjd.sessions._tempdir.os.fstat", + side_effect=self._fstat_reporting_uid(os.geteuid() + 12345), # type: ignore + ): + with pytest.raises(RuntimeError, match="owned by uid"): + custom_gettempdir() + + def test_windows_validator_accepts_a_real_directory(self, tmp_path: Path) -> None: + """The Windows branch must not try to `os.open()` a directory. + + Windows returns EACCES for `os.open()` on a directory whatever the flags, + so an earlier version of this validator failed *every* session creation + on Windows. `os.lstat` is portable, so this runs on all platforms and + pins the branch against a regression back to a descriptor. + """ + # GIVEN a real directory / WHEN validated / THEN it is accepted + root = tmp_path / "OpenJD" + root.mkdir() + _prepare_temp_dir_root_windows(str(root)) + + def test_a_non_posix_platform_never_opens_a_descriptor(self, tmp_path: Path) -> None: + """The dispatcher must not send Windows down the descriptor path. + + This is the bug itself: `os.open()` on a directory returns EACCES on + Windows whatever the flags, so routing Windows through the POSIX + validator failed *every* session creation there. Pinned by patching + `is_posix` rather than by running on Windows, so a POSIX CI host catches + the regression too -- without this, the only signal is a Windows job. + """ + # GIVEN a real directory, and a platform that reports as non-POSIX + root = tmp_path / "OpenJD" + root.mkdir() + + # WHEN validated + with patch("openjd.sessions._tempdir.is_posix", return_value=False): + with patch("openjd.sessions._tempdir.os.open") as opener: + _prepare_temp_dir_root(str(root)) + + # THEN no descriptor was ever taken + opener.assert_not_called() + + @pytest.mark.skipif( + not is_posix(), reason="forces the POSIX branch, whose O_NOFOLLOW/O_DIRECTORY are absent" + ) + def test_a_posix_platform_still_uses_a_descriptor(self, tmp_path: Path) -> None: + """Negative control for the test above: POSIX must keep the fd path. + + The fd path is what makes the symlink-swap window unexploitable, so a + mutation that routed *everything* to the weaker `lstat` validator has to + fail something. + + POSIX-only, and not merely by preference: it drives the POSIX branch + directly, and that branch names `os.O_NOFOLLOW`/`os.O_DIRECTORY`, which + do not exist on every Windows build. The mutation this controls for is + only ever evaluated on a POSIX host anyway. + """ + # GIVEN a real directory on a platform that reports as POSIX + root = tmp_path / "OpenJD" + root.mkdir() + real_open = os.open + opened: list[str] = [] + + def recording_open(path: Any, *a: Any, **k: Any) -> int: + opened.append(str(path)) + return real_open(path, *a, **k) + + # WHEN validated + with patch("openjd.sessions._tempdir.is_posix", return_value=True): + with patch("openjd.sessions._tempdir.os.open", side_effect=recording_open): + _prepare_temp_dir_root(str(root)) + + # THEN the root was validated through a descriptor + assert str(root) in opened + + def test_windows_validator_rejects_a_non_directory(self, tmp_path: Path) -> None: + # GIVEN a plain file where the root should be + squatter = tmp_path / "OpenJD" + squatter.write_text("squat") + + # WHEN / THEN + with pytest.raises(RuntimeError, match="real directory"): + _prepare_temp_dir_root_windows(str(squatter)) + + @pytest.mark.skipif(not is_posix(), reason="symlink creation is POSIX-only here") + def test_windows_validator_rejects_a_symlink(self, tmp_path: Path) -> None: + """`lstat` must not traverse the link, or a directory symlink would pass. + + Exercised with a POSIX symlink because that is what this host can create; + on Windows the same `S_ISDIR` test rejects a directory symlink and + `st_reparse_tag` rejects a junction. + """ + # GIVEN + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + link = tmp_path / "OpenJD" + link.symlink_to(elsewhere, target_is_directory=True) + + # WHEN / THEN + with pytest.raises(RuntimeError, match="real directory"): + _prepare_temp_dir_root_windows(str(link)) + + def test_windows_validator_reports_an_unreadable_root(self, tmp_path: Path) -> None: + # GIVEN a root that cannot be inspected at all + missing = tmp_path / "OpenJD" + + # WHEN / THEN: the failure names the path rather than escaping as OSError + with pytest.raises(RuntimeError, match="could not be inspected"): + _prepare_temp_dir_root_windows(str(missing)) + + @pytest.mark.skipif(not is_posix(), reason="uid ownership check is POSIX-only") + def test_accepts_a_root_owned_by_root(self, tmp_path: Path) -> None: + """A system-provisioned root must keep working.""" + # GIVEN + parent = tmp_path / "parent" + parent.mkdir() + (parent / "OpenJD").mkdir() + + # WHEN / THEN: no exception + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with patch( + "openjd.sessions._tempdir.os.fstat", side_effect=self._fstat_reporting_uid(0) + ): + assert custom_gettempdir() == str(parent / "OpenJD") + + +class TestCleanupErrorReporting: + """R5-7: the old handler accepted the exception and discarded it, leaving a + list of bare paths -- on the one code path where "permission denied" versus + "a process still holds this open" changes what the operator must do.""" + + def test_failure_message_names_the_path_and_the_reason(self, tmp_path: Path) -> None: + # GIVEN: a temp dir whose removal fails with a specific, diagnosable error + d = TempDir(dir=tmp_path) + doomed = d.path / "stubborn.txt" + doomed.write_text("x") + + def boom(path: Any, *a: Any, **k: Any) -> None: + raise PermissionError(13, "Permission denied") + + # WHEN + with ( + patch("openjd.sessions._tempdir.os.unlink", side_effect=boom), + patch("openjd.sessions._tempdir.os.remove", side_effect=boom, create=True), + ): + with pytest.raises(RuntimeError) as excinfo: + d.cleanup() + + # THEN: both the path and the cause are in the message. + message = str(excinfo.value) + assert "stubborn.txt" in message + assert "PermissionError" in message + + def test_successful_cleanup_still_removes_everything(self, tmp_path: Path) -> None: + # GIVEN + d = TempDir(dir=tmp_path) + (d.path / "a.txt").write_text("x") + (d.path / "sub").mkdir() + (d.path / "sub" / "b.txt").write_text("y") + + # WHEN + d.cleanup() + + # THEN + assert not d.path.exists() + + +@pytest.mark.skipif(not is_posix(), reason="symlink swap and fchmod semantics are POSIX here") +class TestTempRootCheckThenUse: + """The first R5-3 implementation validated with `lstat(path)` and then called + `stat(path)`/`chmod(path)`, both of which re-resolve the name and follow + links. Swapping the entry for a symlink in between defeated the check and + chmod'ed the link's target to 0o755.""" + + def test_a_symlink_swapped_in_before_the_open_is_refused(self, tmp_path: Path) -> None: + """Swap during the create window: O_NOFOLLOW must refuse the open.""" + # GIVEN: a victim directory at 0o700 and a parent where the root will live + victim = tmp_path / "victim" + victim.mkdir() + os.chmod(victim, 0o700) + parent = tmp_path / "parent" + parent.mkdir() + root = parent / "OpenJD" + + real_makedirs = os.makedirs + + def makedirs_then_swap(path: Any, *a: Any, **k: Any) -> None: + real_makedirs(path, *a, **k) + if str(path) == str(root): + os.rmdir(root) + os.symlink(victim, root, target_is_directory=True) + + # WHEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with patch("openjd.sessions._tempdir.os.makedirs", side_effect=makedirs_then_swap): + with pytest.raises(RuntimeError): + custom_gettempdir() + + # THEN + assert stat.S_IMODE(os.stat(victim).st_mode) == 0o700 + + def test_a_symlink_swapped_in_after_the_open_cannot_redirect_the_chmod( + self, tmp_path: Path + ) -> None: + """Swap *after* the descriptor is open, which O_NOFOLLOW cannot help with. + + This is the case that pins fchmod-vs-chmod. An earlier version of this + test swapped during the create window instead, where O_NOFOLLOW refuses + the open before any chmod is attempted -- so it passed even with + `os.chmod(path)` restored, and pinned nothing about the descriptor. The + swap is injected from inside `os.fstat`, which the implementation calls + between the open and the mode change. + """ + # GIVEN: a victim at 0o700, and a real root whose mode needs correcting + victim = tmp_path / "victim_after" + victim.mkdir() + os.chmod(victim, 0o700) + parent = tmp_path / "parent_after" + parent.mkdir() + root = parent / "OpenJD" + root.mkdir() + os.chmod(root, 0o700) # != OPENJD_TEMPDIR_MODE, so a mode change is due + + real_fstat = os.fstat + swapped = {"done": False} + + def fstat_then_swap(fd: int, *a: Any, **k: Any) -> Any: + result = real_fstat(fd, *a, **k) + if not swapped["done"]: + swapped["done"] = True + os.rmdir(root) + os.symlink(victim, root, target_is_directory=True) + return result + + # WHEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with patch("openjd.sessions._tempdir.os.fstat", side_effect=fstat_then_swap): + custom_gettempdir() + + # THEN: the swap happened, and the victim was NOT widened. With + # `os.chmod(temp_dir, ...)` restored this is 0o755. + assert swapped["done"] is True + assert os.path.islink(root) + assert stat.S_IMODE(os.stat(victim).st_mode) == 0o700 + + def test_the_validated_inode_is_the_one_modified(self, tmp_path: Path) -> None: + """Even when the swap is not detected as an error, the mode must land on + the inode that was validated, never on a later resolution of the name.""" + # GIVEN: an existing root at a wrong mode, plus a victim + victim = tmp_path / "victim2" + victim.mkdir() + os.chmod(victim, 0o700) + parent = tmp_path / "parent2" + parent.mkdir() + root = parent / "OpenJD" + root.mkdir() + os.chmod(root, 0o700) # differs from OPENJD_TEMPDIR_MODE, so a chmod is due + + # WHEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + result = custom_gettempdir() + + # THEN: the real root got the mode; the victim was not touched. + assert result == str(root) + assert stat.S_IMODE(os.stat(root).st_mode) == stat.S_IMODE(OPENJD_TEMPDIR_MODE) + assert stat.S_IMODE(os.stat(victim).st_mode) == 0o700 + + def test_a_symlinked_root_is_still_refused(self, tmp_path: Path) -> None: + """O_NOFOLLOW must keep doing what the explicit S_ISLNK branch did.""" + # GIVEN + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + parent = tmp_path / "parent3" + parent.mkdir() + (parent / "OpenJD").symlink_to(elsewhere, target_is_directory=True) + + # WHEN / THEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with pytest.raises(RuntimeError, match="real directory"): + custom_gettempdir() + + def test_no_descriptor_is_leaked(self, tmp_path: Path) -> None: + """The validation opens a descriptor; every path must close it. + + Probe: the lowest free descriptor number. If a call leaks a descriptor, + the number the kernel hands out next goes up. Portable across macOS and + Linux, unlike listing /dev/fd. + """ + # GIVEN + parent = tmp_path / "parent4" + parent.mkdir() + (parent / "OpenJD").mkdir() + + def lowest_free_fd() -> int: + """Probe the lowest free descriptor number by taking and releasing one. + + A `with` block is not usable here: the number itself is the + measurement, so the descriptor has to be closed before it is returned. + `try/finally` guarantees that on every path. + """ + fd = os.open(os.devnull, os.O_RDONLY) + try: + return fd + finally: + os.close(fd) + + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + custom_gettempdir() # warm up, so first-call allocations do not count + baseline = lowest_free_fd() + + # WHEN: many successful calls + for _ in range(25): + custom_gettempdir() + after_success = lowest_free_fd() + + # WHEN: many calls that fail *after* the descriptor is opened + with patch( + "openjd.sessions._tempdir.os.fstat", side_effect=OSError(5, "induced failure") + ): + for _ in range(25): + with pytest.raises(OSError): + custom_gettempdir() + after_failure = lowest_free_fd() + + # THEN: no drift on either path. + assert after_success == baseline + assert after_failure == baseline diff --git a/test/openjd/sessions_v0/test_wrap_actions.py b/test/openjd/sessions_v0/test_wrap_actions.py new file mode 100644 index 00000000..f3fd5b5e --- /dev/null +++ b/test/openjd/sessions_v0/test_wrap_actions.py @@ -0,0 +1,710 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""End-to-end tests for the three RFC 0008 wrap hooks +(``onWrapEnvEnter``, ``onWrapTaskRun``, ``onWrapEnvExit``) and the +single-wrap-layer validation. + +The tests use the marker-file pattern from the RFC: each action +appends a tagged line to a shared file in the session working +directory, and the final contents prove which actions ran via which +path. No containers required — ``echo`` and ``cat`` only. +""" + +from __future__ import annotations + +import time +import uuid +from pathlib import Path + +import pytest + +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CommandString as CommandString_2023_09, + Environment as Environment_2023_09, + EnvironmentActions as EnvironmentActions_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, + StepActions as StepActions_2023_09, + StepScript as StepScript_2023_09, +) +from openjd.sessions import ActionState, ActionStatus, Session, SessionState + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _action(command: str, *args: str) -> Action_2023_09: + return Action_2023_09( + command=CommandString_2023_09(command), + args=[ArgString_2023_09(a) for a in args] if args else None, + ) + + +def _env(name: str, **action_kwargs) -> Environment_2023_09: + """Build an environment whose script defines whichever hooks the + caller passes (e.g. ``onEnter=…``, ``onWrapTaskRun=…``).""" + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09(**action_kwargs), + ), + ) + + +def _step(command: str, *args: str) -> StepScript_2023_09: + return StepScript_2023_09(actions=StepActions_2023_09(onRun=_action(command, *args))) + + +def _run_until_ready(session: Session, timeout_s: float = 10.0) -> None: + deadline = time.time() + timeout_s + while session.state == SessionState.RUNNING and time.time() < deadline: + time.sleep(0.05) + + +def _trace_path(session: Session) -> Path: + """The file the test scripts append to. + + Interpolate it into a shell command as ``'{trace.as_posix()}'`` -- quoted and + slash-separated. A native Windows path interpolated bare has its backslashes + consumed as escapes by the ``sh`` these tests invoke, so the redirect would + silently land somewhere else. + """ + return Path(str(session.working_directory)) / "trace.log" + + +_NOOP = _action("true") + + +# --------------------------------------------------------------------------- +# Single-wrap-layer enforcement (RFC 0008: at most one wrap-defining +# environment in the session stack). +# --------------------------------------------------------------------------- + + +class TestSingleWrapLayer: + def test_two_wrap_envs_rejected_at_enter(self) -> None: + outer = _env( + "outer", + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_action("sh", "-c", "echo outer-wrap"), + onWrapEnvExit=_NOOP, + ) + inner = _env( + "inner", + onWrapEnvEnter=_action("sh", "-c", "echo inner-wrap"), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=outer) + _run_until_ready(session) + + with pytest.raises(RuntimeError, match=r"RFC 0008"): + session.enter_environment(environment=inner) + + def test_two_envs_with_only_one_wrap_layer_ok(self) -> None: + outer = _env( + "outer", + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_action("sh", "-c", "echo outer-wrap"), + onWrapEnvExit=_NOOP, + ) + # The inner env defines no wrap hooks — fine to enter. + inner = _env("inner", onEnter=_action("sh", "-c", "echo inner-onEnter")) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=outer) + _run_until_ready(session) + session.enter_environment(environment=inner) + _run_until_ready(session) + assert session.state == SessionState.READY + + +# --------------------------------------------------------------------------- +# onWrapEnvEnter intercepts inner onEnter (RFC Test 1). +# --------------------------------------------------------------------------- + + +class TestWrapEnvEnter: + def test_wrap_env_enter_intercepts_inner_on_enter(self) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + trace = _trace_path(session) + outer = _env( + "outer", + onWrapEnvEnter=_action( + "sh", + "-c", + f"echo '[WRAPPED] inner-onEnter' >> '{trace.as_posix()}' && " + f"echo 'inner-onEnter body' >> '{trace.as_posix()}'", + ), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + onEnter=_action("sh", "-c", f"echo 'outer-onEnter' >> '{trace.as_posix()}'"), + ) + inner = _env( + "inner", + onEnter=_action( + "sh", "-c", f"echo 'should not run on host' >> '{trace.as_posix()}'" + ), + ) + + session.enter_environment(environment=outer) + _run_until_ready(session) + session.enter_environment(environment=inner) + _run_until_ready(session) + + assert session.state == SessionState.READY + assert trace.exists() + content = trace.read_text() + # The outer env's own onEnter ran on host. + assert "outer-onEnter" in content + # The wrap script ran in place of the inner onEnter. + assert "[WRAPPED] inner-onEnter" in content + assert "inner-onEnter body" in content + # The inner env's host-side onEnter did NOT run. + assert "should not run on host" not in content + + def test_wrap_env_enter_receives_wrapped_symbols(self) -> None: + """``WrappedEnv.Name`` and ``WrappedAction.Command`` resolve to the + inner env's identity inside the wrap script.""" + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + trace = _trace_path(session) + outer = _env( + "outer", + onWrapEnvEnter=_action( + "sh", + "-c", + "echo " + "'name={{WrappedEnv.Name}} cmd={{WrappedAction.Command}}' " + f">> '{trace.as_posix()}'", + ), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + inner = _env("inner-env-name", onEnter=_action("echo", "hello")) + + session.enter_environment(environment=outer) + _run_until_ready(session) + session.enter_environment(environment=inner) + _run_until_ready(session) + + assert session.state == SessionState.READY + content = trace.read_text() + assert "name=inner-env-name" in content + assert "cmd=echo" in content + + +# --------------------------------------------------------------------------- +# onWrapEnvExit intercepts inner onExit (RFC Test 2). +# --------------------------------------------------------------------------- + + +class TestWrapEnvExit: + def test_wrap_env_exit_intercepts_inner_on_exit(self) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + trace = _trace_path(session) + outer = _env( + "outer", + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_NOOP, + onWrapEnvExit=_action( + "sh", + "-c", + f"echo '[WRAPPED] inner-onExit' >> '{trace.as_posix()}'", + ), + onExit=_action("sh", "-c", f"echo 'outer-onExit' >> '{trace.as_posix()}'"), + ) + inner = _env( + "inner", + onExit=_action( + "sh", "-c", f"echo 'should not run on host' >> '{trace.as_posix()}'" + ), + ) + + outer_id = session.enter_environment(environment=outer) + _run_until_ready(session) + inner_id = session.enter_environment(environment=inner) + _run_until_ready(session) + + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + + content = trace.read_text() + assert "[WRAPPED] inner-onExit" in content + assert "should not run on host" not in content + + # The outer env's own onExit must still run on host when we + # eventually exit it. + session.exit_environment(identifier=outer_id) + _run_until_ready(session) + content = trace.read_text() + assert "outer-onExit" in content + + +# --------------------------------------------------------------------------- +# Visible end-to-end ordering across all three hooks (RFC Test 5). +# --------------------------------------------------------------------------- + + +class TestVisibleOrdering: + def test_full_ordering_across_three_hooks(self) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + trace = _trace_path(session) + outer = _env( + "outer", + onEnter=_action("sh", "-c", f"echo 'outer-onEnter' >> '{trace.as_posix()}'"), + onWrapEnvEnter=_action( + "sh", "-c", f"echo '[WRAPPED] inner-onEnter' >> '{trace.as_posix()}'" + ), + onWrapTaskRun=_action( + "sh", "-c", f"echo '[WRAPPED] task-onRun' >> '{trace.as_posix()}'" + ), + onWrapEnvExit=_action( + "sh", "-c", f"echo '[WRAPPED] inner-onExit' >> '{trace.as_posix()}'" + ), + onExit=_action("sh", "-c", f"echo 'outer-onExit' >> '{trace.as_posix()}'"), + ) + wrapped_inner = _env( + "wrapped-inner", + onEnter=_action("sh", "-c", f"echo 'inner-onEnter body' >> '{trace.as_posix()}'"), + onExit=_action("sh", "-c", f"echo 'inner-onExit body' >> '{trace.as_posix()}'"), + ) + step = _step("echo", "task-onRun-body") + + outer_id = session.enter_environment(environment=outer) + _run_until_ready(session) + wrapped_id = session.enter_environment(environment=wrapped_inner) + _run_until_ready(session) + + session.run_task(step_script=step, task_parameter_values={}, step_name="Step") + _run_until_ready(session) + + session.exit_environment(identifier=wrapped_id) + _run_until_ready(session) + session.exit_environment(identifier=outer_id) + _run_until_ready(session) + + lines = [line for line in trace.read_text().splitlines() if line.strip()] + # The exact order matters here — RFC 0008's pass criterion. + assert lines[0] == "outer-onEnter" + assert lines[1] == "[WRAPPED] inner-onEnter" + assert lines[2] == "[WRAPPED] task-onRun" + assert lines[3] == "[WRAPPED] inner-onExit" + assert lines[4] == "outer-onExit" + + +# --------------------------------------------------------------------------- +# WrappedAction.* injection: inner embedded files, and failure handling. +# +# openjd-rs #277: the wrapped entity's embedded files ARE materialized on +# the wrap path (into the inner scope only), so a wrapped action +# referencing {{Task.File.*}} / {{Env.File.*}} resolves to a real on-disk +# path in WrappedAction.Command. Injection failures that remain possible +# (e.g. an undefined symbol) must still FAIL the action through the normal +# callback path instead of raising out of the public API, and the session +# transitions to READY_ENDING (never stuck in RUNNING). +# --------------------------------------------------------------------------- + + +def _embedded_file_step() -> StepScript_2023_09: + from openjd.model.v2023_09 import ( + DataString as DataString_2023_09, + EmbeddedFileText as EmbeddedFileText_2023_09, + EmbeddedFileTypes as EmbeddedFileTypes_2023_09, + ) + + return StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09(command=CommandString_2023_09("{{ Task.File.Foo }}")) + ), + embeddedFiles=[ + EmbeddedFileText_2023_09( + name="Foo", + type=EmbeddedFileTypes_2023_09.TEXT, + runnable=True, + data=DataString_2023_09("#!/bin/sh\necho hello\n"), + ) + ], + ) + + +def _embedded_file_env(name: str) -> Environment_2023_09: + from openjd.model.v2023_09 import ( + DataString as DataString_2023_09, + EmbeddedFileText as EmbeddedFileText_2023_09, + EmbeddedFileTypes as EmbeddedFileTypes_2023_09, + ) + + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09(command=CommandString_2023_09("{{ Env.File.Setup }}")), + ), + embeddedFiles=[ + EmbeddedFileText_2023_09( + name="Setup", + type=EmbeddedFileTypes_2023_09.TEXT, + runnable=True, + data=DataString_2023_09("#!/bin/sh\necho setup\n"), + ) + ], + ), + ) + + +class TestWrapInjectionFailure: + def _wrap_env(self) -> Environment_2023_09: + return _env( + "Wrapper", + onWrapEnvEnter=_action("echo", "{{WrappedAction.Command}}"), + onWrapTaskRun=_action("echo", "{{WrappedAction.Command}}"), + onWrapEnvExit=_NOOP, + ) + + def test_run_task_with_embedded_file_command_resolves( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # openjd-rs #277: a wrapped onRun referencing {{Task.File.*}} now + # resolves — the step's embedded files are materialized into the + # inner scope, and WrappedAction.Command carries the on-disk path. + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=self._wrap_env()) + _run_until_ready(session) + assert session.state == SessionState.READY + + session.run_task( + step_script=_embedded_file_step(), + task_parameter_values={}, + step_name="Step", + ) + _run_until_ready(session) + + assert session.state == SessionState.READY + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + # The wrap hook echoed WrappedAction.Command: the materialized + # file path inside the session's files directory. + messages = "\n".join(caplog.messages) + assert str(session.files_directory) in messages + + def test_enter_environment_with_embedded_file_on_enter_resolves( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # openjd-rs #277: a wrapped onEnter referencing {{Env.File.*}} now + # resolves against the inner environment's own materialized files. + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=self._wrap_env()) + _run_until_ready(session) + + inner_id = session.enter_environment(environment=_embedded_file_env("Inner")) + _run_until_ready(session) + + assert session.state == SessionState.READY + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + messages = "\n".join(caplog.messages) + assert str(session.files_directory) in messages + session.exit_environment(identifier=inner_id, keep_session_running=True) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + def test_run_task_with_unresolvable_wrapped_symbol_fails_action(self) -> None: + # The graceful-failure contract still holds for injection failures: + # a wrapped onRun referencing an undefined symbol must FAIL the + # action via the callback path, not raise out of run_task(). + bad_step = StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09(command=CommandString_2023_09("{{ Task.File.DoesNotExist }}")) + ), + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=self._wrap_env()) + _run_until_ready(session) + assert session.state == SessionState.READY + + session.run_task( + step_script=bad_step, + task_parameter_values={}, + step_name="Step", + ) + _run_until_ready(session) + + assert session.state == SessionState.READY_ENDING + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert status.fail_message is not None + assert "onWrapTaskRun" in status.fail_message + + +class TestWrapScopeSeparation: + """openjd-rs #277: the wrapper's and the wrapped entity's scopes must be + kept strictly apart. WrappedAction.* resolves against the INNER scope + only; the hook script resolves against the WRAP environment's own scope + (including its script-level ``let`` bindings) only.""" + + def test_same_let_name_each_side_sees_own_value(self, caplog: pytest.LogCaptureFixture) -> None: + # GIVEN: the wrapper and the step both bind the same `let` name + # `who` to different values. The step's onRun args reference + # {{who}}; the hook also references {{who}} directly. + wrapper = Environment_2023_09( + name="Wrapper", + script=EnvironmentScript_2023_09( + let=["who = 'wrapper-scope'"], + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_action( + "sh", + "-c", + "echo HOOK-WHO={{who}} WRAPPED-ARGS={{WrappedAction.Args}}", + ), + onWrapEnvExit=_NOOP, + ), + ), + ) + step = StepScript_2023_09( + let=["who = 'step-scope'"], + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("{{who}}")], + ) + ), + ) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=wrapper) + _run_until_ready(session) + + # WHEN: the task runs through the wrap hook. + session.run_task(step_script=step, task_parameter_values={}, step_name="Step") + _run_until_ready(session) + + # THEN: the hook saw the WRAPPER's binding, and the wrapped + # action's args resolved with the STEP's binding. + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + messages = "\n".join(caplog.messages) + assert "HOOK-WHO=wrapper-scope" in messages, messages + # WrappedAction.Args is a list; its default rendering brackets the + # single resolved element. + assert "WRAPPED-ARGS=[step-scope]" in messages, messages + + def test_inner_env_let_does_not_leak_into_hook_scope(self) -> None: + # An inner environment's `let` binding must resolve in + # WrappedAction.* but must NOT be visible to the hook script: a + # hook referencing it directly fails the action. + wrapper = _env( + "Wrapper", + onWrapEnvEnter=_action("sh", "-c", "echo LEAKED={{inner_only}}"), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + inner = Environment_2023_09( + name="Inner", + script=EnvironmentScript_2023_09( + let=["inner_only = 'secret'"], + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09(command=CommandString_2023_09("{{inner_only}}")), + ), + ), + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=wrapper) + _run_until_ready(session) + + inner_id = session.enter_environment(environment=inner) + _run_until_ready(session) + + # The hook referenced {{inner_only}}, which is not in its scope. + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + session.exit_environment(identifier=inner_id, keep_session_running=True) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + def test_wrapped_env_enter_command_resolves_inner_let( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # The inner environment's `let` binding must be visible to + # WrappedAction.* on the onWrapEnvEnter path. + wrapper = _env( + "Wrapper", + onWrapEnvEnter=_action("sh", "-c", "echo INNERCMD={{WrappedAction.Command}}"), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + inner = Environment_2023_09( + name="Inner", + script=EnvironmentScript_2023_09( + let=["tool = 'inner-tool'"], + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09(command=CommandString_2023_09("{{tool}}")), + ), + ), + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=wrapper) + _run_until_ready(session) + + inner_id = session.enter_environment(environment=inner) + _run_until_ready(session) + + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + session.exit_environment(identifier=inner_id, keep_session_running=True) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + messages = "\n".join(caplog.messages) + assert "INNERCMD=inner-tool" in messages, messages + + +class TestWrappedActionEnvironmentContents: + def test_variables_map_included_in_wrapped_environment(self) -> None: + # RFC 0008 (openjd-rs #277): WrappedAction.Environment carries every + # session-defined variable — openjd_env definitions AND entered + # environments' declarative variables: maps. Host-inherited + # variables remain excluded. + from openjd.model.v2023_09 import ( + EnvironmentVariableValueString as EnvironmentVariableValueString_2023_09, + ) + + declaring = Environment_2023_09( + name="Declaring", + variables={ + "DECLARED_VAR": EnvironmentVariableValueString_2023_09("from-variables-map") + }, + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment( + environment=_env( + "Wrapper", + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + ) + _run_until_ready(session) + session.enter_environment(environment=declaring) + _run_until_ready(session) + + env_list = session._collect_session_env_list() + assert any(entry == "DECLARED_VAR=from-variables-map" for entry in env_list), env_list + + def test_later_set_overrides_earlier_value_without_duplicate(self) -> None: + # Cumulative flattening: when a later environment's openjd_env sets + # a name an earlier environment already set, the list carries only + # the effective (later) value — matching the real subprocess env + # and the Rust runtime's single cumulative map. + from openjd.sessions._session import ( + EnvironmentVariableSetChange, + EnvironmentVariableUnsetChange, + SimplifiedEnvironmentVariableChanges, + ) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + scenarios: list[ + tuple[str, list[EnvironmentVariableSetChange | EnvironmentVariableUnsetChange]] + ] = [ + ("env-a", [EnvironmentVariableSetChange(name="FOO", value="1")]), + ("env-b", [EnvironmentVariableSetChange(name="FOO", value="2")]), + ] + for env_id, changes in scenarios: + session._environments_entered.append(env_id) + tracked = SimplifiedEnvironmentVariableChanges({}) + tracked.simplify_ordered_changes(changes) + session._created_env_vars[env_id] = tracked + + assert session._collect_session_env_list() == ["FOO=2"] + + # And a later unset removes the name entirely. + session._environments_entered.append("env-c") + tracked = SimplifiedEnvironmentVariableChanges({}) + tracked.simplify_ordered_changes([EnvironmentVariableUnsetChange(name="FOO")]) + session._created_env_vars["env-c"] = tracked + + assert session._collect_session_env_list() == [] + + # Clear the fake stack so Session.cleanup() doesn't try to + # unwind environments that were never really entered. + session._environments_entered.clear() + session._created_env_vars.clear() + + def test_exit_path_includes_exiting_envs_openjd_env_vars(self) -> None: + # On the onWrapEnvExit path, WrappedAction.Environment must include + # the exiting environment's own openjd_env variables: the real + # subprocess env (and the unwrapped onExit) includes them, so the + # list must match. The wrapper's onWrapEnvEnter emits the + # openjd_env message, which is attributed to the inner (wrapped) + # environment being entered. + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + trace = _trace_path(session) + wrapper = _env( + "Wrapper", + onWrapEnvEnter=_action("sh", "-c", "echo 'openjd_env: INNER_VAR=inner-value'"), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_action( + "sh", + "-c", + f"echo \"ENVLIST=<{{{{WrappedAction.Environment}}}}>\" >> '{trace.as_posix()}'", + ), + ) + inner = _env("Inner", onEnter=_NOOP, onExit=_NOOP) + + wrap_id = session.enter_environment(environment=wrapper) + _run_until_ready(session) + inner_id = session.enter_environment(environment=inner) + _run_until_ready(session) + session.exit_environment(identifier=inner_id, keep_session_running=True) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + contents = trace.read_text() + assert "INNER_VAR=inner-value" in contents, contents + + def test_openjd_env_vars_included_in_wrapped_environment(self) -> None: + # openjd_env-emitted variables must still be surfaced. The wrapper's + # own onEnter is never wrapped, so it can emit the message directly. + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment( + environment=_env( + "Wrapper", + onEnter=_action("sh", "-c", "echo 'openjd_env: EMITTED_VAR=from-openjd-env'"), + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + ) + _run_until_ready(session) + + env_list = session._collect_session_env_list() + assert "EMITTED_VAR=from-openjd-env" in env_list + + +class TestRunWrapHookGuard: + def test_unknown_hook_name_raises(self) -> None: + from openjd.sessions._runner_env_script import EnvironmentScriptRunner + import logging + + from openjd.sessions._logging import LoggerAdapter + from openjd.model import SymbolTable + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + runner = EnvironmentScriptRunner( + logger=LoggerAdapter(logging.getLogger(__name__), extra={}), + session_working_directory=Path(str(session.working_directory)), + environment_script=None, + symtab=SymbolTable(), + session_files_directory=Path(str(session.files_directory)), + ) + with pytest.raises(ValueError, match="Unknown wrap hook name"): + runner._run_wrap_hook("onWrapTypo") diff --git a/test/openjd/sessions_v0/test_wrap_cancelation.py b/test/openjd/sessions_v0/test_wrap_cancelation.py new file mode 100644 index 00000000..ecc021eb --- /dev/null +++ b/test/openjd/sessions_v0/test_wrap_cancelation.py @@ -0,0 +1,566 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for ``WrappedAction.Cancelation.*`` injection (RFC 0008 follow-up, +openjd-specifications#148). + +``WrappedAction.Cancelation.Mode`` is ``string?`` and carries the wrapped +action's cancelation method — ``"TERMINATE"``, ``"NOTIFY_THEN_TERMINATE"``, +or ``None`` when the wrapped action defines no ````. The null +case is deliberately distinct from an explicit ``TERMINATE``. + +``WrappedAction.Cancelation.NotifyPeriodInSeconds`` is ``int?``: the +effective grace period when the mode is ``NOTIFY_THEN_TERMINATE`` (with the +Template Schemas 5.3.2 defaults applied — 120 for a task's ``onRun``, 30 +otherwise), and ``None`` when a notify period does not apply. + +Mirrors the Rust integration coverage in openjd-rs +``tests/integration/test_wrap_actions.rs`` and the conformance fixtures +``conformance-tests/2023-09/WRAP_ACTIONS/jobs/wrap-cancelation-*``. +""" + +from __future__ import annotations + +import time +import uuid + +import pytest + +from openjd.model import SymbolTable +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CancelationMethodNotifyThenTerminate as NotifyThenTerminate_2023_09, + CancelationMethodTerminate as Terminate_2023_09, + CancelationMode as CancelationMode_2023_09, + CommandString as CommandString_2023_09, + Environment as Environment_2023_09, + EnvironmentActions as EnvironmentActions_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, + StepActions as StepActions_2023_09, + StepScript as StepScript_2023_09, +) +from openjd.sessions import ActionState, ActionStatus, Session, SessionState +from openjd.sessions._os_checker import is_posix + +from .conftest import serial_process + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +_NOOP = Action_2023_09(command=CommandString_2023_09("true")) + +_TERMINATE = Terminate_2023_09(mode=CancelationMode_2023_09.TERMINATE) + + +def _notify(period: int | None) -> NotifyThenTerminate_2023_09: + return NotifyThenTerminate_2023_09( + mode=CancelationMode_2023_09.NOTIFY_THEN_TERMINATE, + notifyPeriodInSeconds=period, + ) + + +def _step_script_with_cancelation(cancelation) -> StepScript_2023_09: + return StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("placeholder")], + cancelation=cancelation, + ) + ) + ) + + +def _wrap_env(name: str, wrap_action: Action_2023_09) -> Environment_2023_09: + """Build an Environment with ``onWrapTaskRun`` set to ``wrap_action`` + and the other two wrap hooks set to no-ops (all-or-nothing rule).""" + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=_NOOP, + onWrapTaskRun=wrap_action, + onWrapEnvExit=_NOOP, + ), + ), + ) + + +def _inner_env_action(cancelation) -> Action_2023_09: + return Action_2023_09( + command=CommandString_2023_09("inner-enter-cmd"), + args=[ArgString_2023_09("--flag")], + cancelation=cancelation, + ) + + +def _run_until_ready(session: Session, timeout_s: float = 10.0) -> None: + deadline = time.time() + timeout_s + while session.state == SessionState.RUNNING and time.time() < deadline: + time.sleep(0.05) + + +# --------------------------------------------------------------------------- +# Unit tests — task-run injection path +# --------------------------------------------------------------------------- + + +class TestInjectTaskCancelationSymbols: + """Unit tests for cancelation injection via ``_inject_wrapped_task_symbols``.""" + + def _inject(self, cancelation) -> SymbolTable: + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + script = _step_script_with_cancelation(cancelation) + session._inject_wrapped_task_symbols(symtab, script, "Step1", inner_symtab=symtab) + return symtab + finally: + session.cleanup() + + def test_mode_none_and_period_none_when_no_cancelation(self) -> None: + # No declared: Mode is None (NOT "TERMINATE") and the + # notify period is None — the string?/int? null values. + symtab = self._inject(None) + assert symtab["WrappedAction.Cancelation.Mode"] is None + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] is None + + def test_mode_terminate_and_period_none(self) -> None: + symtab = self._inject(_TERMINATE) + assert symtab["WrappedAction.Cancelation.Mode"] == "TERMINATE" + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] is None + + def test_notify_then_terminate_with_explicit_period(self) -> None: + symtab = self._inject(_notify(45)) + assert symtab["WrappedAction.Cancelation.Mode"] == "NOTIFY_THEN_TERMINATE" + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] == 45 + + def test_notify_then_terminate_defaults_to_120_for_task_on_run(self) -> None: + # Template Schemas 5.3.2: the default notify period for a task's + # onRun is 120 seconds. The runtime supplies the value it would have + # enforced in the unwrapped case. + symtab = self._inject(_notify(None)) + assert symtab["WrappedAction.Cancelation.Mode"] == "NOTIFY_THEN_TERMINATE" + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] == 120 + + +# --------------------------------------------------------------------------- +# Unit tests — env enter/exit injection path +# --------------------------------------------------------------------------- + + +class TestInjectEnvCancelationSymbols: + """Unit tests for cancelation injection via ``_inject_wrapped_env_symbols``.""" + + def _inject(self, cancelation) -> SymbolTable: + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + inner_env = Environment_2023_09( + name="InnerEnv", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=_inner_env_action(cancelation), + ), + ), + ) + session._inject_wrapped_env_symbols( + symtab, inner_env, _inner_env_action(cancelation), inner_symtab=symtab + ) + return symtab + finally: + session.cleanup() + + def test_notify_then_terminate_defaults_to_30_for_env_action(self) -> None: + # Template Schemas 5.3.2: for anything other than a task's onRun — + # including an inner environment's onEnter — the default is 30. + symtab = self._inject(_notify(None)) + assert symtab["WrappedAction.Cancelation.Mode"] == "NOTIFY_THEN_TERMINATE" + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] == 30 + + def test_explicit_period_forwards_verbatim_for_env_action(self) -> None: + symtab = self._inject(_notify(45)) + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] == 45 + + def test_mode_none_when_env_action_has_no_cancelation(self) -> None: + symtab = self._inject(None) + assert symtab["WrappedAction.Cancelation.Mode"] is None + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] is None + + +# --------------------------------------------------------------------------- +# Integration tests — real subprocess, format-string interpolation +# --------------------------------------------------------------------------- + + +class TestWrapCancelationExecution: + """End-to-end: the wrap action interpolates the Cancelation variables in + a format string. The ``<...>`` sentinel wrap makes the null-renders-empty + behavior observable (``NP=<>``), matching the conformance fixtures.""" + + _PROBE = Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09( + 'echo "MODE=<{{WrappedAction.Cancelation.Mode}}>";' + ' echo "NP=<{{WrappedAction.Cancelation.NotifyPeriodInSeconds}}>"' + ), + ], + ) + + def _run(self, cancelation, caplog: pytest.LogCaptureFixture) -> str: + session_id = uuid.uuid4().hex + env = _wrap_env("wrap_env", self._PROBE) + step = _step_script_with_cancelation(cancelation) + with Session(session_id=session_id, job_parameter_values={}) as session: + session.enter_environment(environment=env) + _run_until_ready(session) + session.run_task(step_script=step, task_parameter_values={}, step_name="Step") + _run_until_ready(session) + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + return "\n".join(caplog.messages) + + def test_terminate_mode_renders_null_period_as_empty( + self, caplog: pytest.LogCaptureFixture + ) -> None: + messages = self._run(_TERMINATE, caplog) + assert "MODE=" in messages + # int? null interpolates as the empty string (RFC 0005), so the + # sentinel wrap renders as NP=<> — distinct from the old 0 sentinel, + # which would have rendered NP=<0>. + assert "NP=<>" in messages + + def test_notify_then_terminate_default_renders_120( + self, caplog: pytest.LogCaptureFixture + ) -> None: + messages = self._run(_notify(None), caplog) + assert "MODE=" in messages + assert "NP=<120>" in messages + + def test_no_cancelation_renders_null_mode_as_empty( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # No declared: Mode is null (string?), which + # interpolates as the empty string (RFC 0005) — MODE=<>, matching + # the wrap-cancelation-mode-null-when-no-cancelation conformance + # fixture. The nullness itself (vs. an empty string value) is + # asserted by the unit tests above and observable via EXPR + # null-coalescing in the conformance fixture. + messages = self._run(None, caplog) + assert "MODE=<>" in messages + assert "NP=<>" in messages + + +# --------------------------------------------------------------------------- +# Unit tests — deferred-mode resolution (resolve_effective_cancelation) +# --------------------------------------------------------------------------- + + +class TestResolveEffectiveCancelation: + """Unit tests for the shared deferred-cancelation resolution helper. + + A CancelationMethodDeferred carries a format-string mode whose + TERMINATE-vs-NOTIFY_THEN_TERMINATE decision is made at run time + against the live symbol table (see resolve_effective_cancelation's + docstring for the full story). + """ + + def _deferred(self, mode: str, period: str | None = None): + from openjd.model._format_strings import FormatString + from openjd.model.v2023_09 import CancelationMethodDeferred, ModelParsingContext + + # A deferred mode is an RFC 0008 forwarding construct, and + # WRAP_ACTIONS requires the EXPR extension — so the format strings + # here parse as EXPR expressions, giving the typed null semantics + # the runtime relies on (a whole-field null drops the cancelation + # object; an empty STRING is an error, matching openjd-rs). + ctx = ModelParsingContext(supported_extensions=["FEATURE_BUNDLE_1", "EXPR"]) + return CancelationMethodDeferred( + mode=FormatString(mode, context=ctx), + notifyPeriodInSeconds=( + FormatString(period, context=ctx) if period is not None else None + ), + ) + + def _symtab(self, **values) -> SymbolTable: + symtab = SymbolTable() + for key, value in values.items(): + symtab[key] = value + return symtab + + def test_mode_resolving_null_drops_whole_object(self) -> None: + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}", "{{P}}") + result = resolve_effective_cancelation(cancelation, self._symtab(X=None, P=None)) + assert result == (None, None) + + def test_mode_resolving_terminate(self) -> None: + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}") + result = resolve_effective_cancelation(cancelation, self._symtab(X="TERMINATE")) + assert result == ("TERMINATE", None) + + def test_mode_resolving_terminate_rejects_non_null_period(self) -> None: + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}", "{{P}}") + with pytest.raises(ValueError, match="does not accept"): + resolve_effective_cancelation(cancelation, self._symtab(X="TERMINATE", P=45)) + + def test_mode_resolving_notify_then_terminate_with_period(self) -> None: + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}", "{{P}}") + result = resolve_effective_cancelation( + cancelation, self._symtab(X="NOTIFY_THEN_TERMINATE", P=45) + ) + assert result == ("NOTIFY_THEN_TERMINATE", 45) + + def test_whole_field_mode_resolving_empty_string_raises(self) -> None: + # A genuine empty STRING is not null, even for a whole-field + # expression (openjd-rs parity: only an ExprValue::Null result + # drops the cancelation object; an empty string is an invalid + # mode). E.g. a STRING parameter whose value is "". + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}") + with pytest.raises(ValueError, match="must resolve to .* got ''"): + resolve_effective_cancelation(cancelation, self._symtab(X="")) + + def test_mode_resolving_garbage_raises(self) -> None: + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}") + with pytest.raises(ValueError, match="must resolve to"): + resolve_effective_cancelation(cancelation, self._symtab(X="SOMETHING_ELSE")) + + def test_partial_interpolation_mode_resolves_normally(self) -> None: + # Normal format string behavior (Template Schemas 5.3): partial + # interpolation is permitted; the resolved value is checked + # against the two mode names. + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}_THEN_TERMINATE", "{{P}}") + result = resolve_effective_cancelation(cancelation, self._symtab(X="NOTIFY", P=45)) + assert result == ("NOTIFY_THEN_TERMINATE", 45) + + def test_partial_interpolation_mode_resolving_empty_raises(self) -> None: + # Null semantics (dropping the cancelation object) apply only to a + # whole-field expression. A normal format string that resolves to + # the empty string is not null — it is an invalid mode. + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}{{Y}}") + with pytest.raises(ValueError, match="must resolve to"): + resolve_effective_cancelation(cancelation, self._symtab(X=None, Y=None)) + + def test_resolved_period_exceeding_cap_raises(self) -> None: + # The static validator caps literal periods at 600 (Template + # Schemas 5.3.2); format-string values could not be checked at + # parse time, so the resolved value is bounds-checked at run time. + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}", "{{P}}") + with pytest.raises(ValueError, match="between 1 and 600"): + resolve_effective_cancelation( + cancelation, self._symtab(X="NOTIFY_THEN_TERMINATE", P=9999) + ) + + +# --------------------------------------------------------------------------- +# Launch-time cancelation resolution (openjd-rs run_action parity): +# the effective cancel method is resolved by _run_action against the SAME +# final scope the command/args resolved with (script lets, *.File.*, +# WrappedAction.*) and stored on the runner; cancel() consumes it. An +# unresolvable or invalid cancelation fails the action at start. +# --------------------------------------------------------------------------- + + +class TestLaunchTimeCancelationResolution: + def _env_with_let_bound_cancelation(self) -> Environment_2023_09: + from openjd.model.v2023_09 import ModelParsingContext + + ctx = ModelParsingContext(supported_extensions=["EXPR", "WRAP_ACTIONS", "FEATURE_BUNDLE_1"]) + return Environment_2023_09.model_validate( + { + "name": "Wrapper", + "script": { + "let": [ + "hookMode = 'NOTIFY_THEN_TERMINATE'", + "hookPeriod = 9", + ], + "actions": { + "onEnter": {"command": "true"}, + "onWrapEnvEnter": {"command": "true"}, + "onWrapEnvExit": {"command": "true"}, + "onWrapTaskRun": { + "command": "sleep", + "args": ["20"], + "cancelation": { + "mode": "{{hookMode}}", + "notifyPeriodInSeconds": "{{hookPeriod}}", + }, + }, + }, + }, + }, + context=ctx, + ) + + def test_let_bound_cancelation_resolved_against_final_scope(self) -> None: + # Regression: cancel() used to re-resolve the cancelation against + # the runner's BASE symtab — which lacks the script's `let` + # bindings — so a let-referencing mode fell back to Terminate with + # a warning. It must resolve at launch, in the hook's final scope. + from datetime import timedelta + from unittest.mock import MagicMock, patch + + from openjd.sessions._runner_env_script import EnvironmentScriptRunner + from openjd.sessions._runner_base import NotifyCancelMethod + + env = self._env_with_let_bound_cancelation() + import pathlib + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + with patch.object(EnvironmentScriptRunner, "_run"): + runner = EnvironmentScriptRunner( + logger=MagicMock(), + session_working_directory=tmp_path, + environment_script=env.script, + symtab=SymbolTable(), + session_files_directory=tmp_path, + ) + runner.wrap_task_run() + assert runner._resolved_cancel_method == NotifyCancelMethod( + terminate_delay=timedelta(seconds=9) + ) + + def test_invalid_deferred_mode_fails_action_at_launch(self) -> None: + # Eager validation (openjd-rs parity): a cancelation whose deferred + # mode resolves to something other than the two method names or + # null must FAIL the action at start — not launch successfully and + # only surface if a cancel later occurs. + session_id = uuid.uuid4().hex + probe = Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ArgString_2023_09("-c"), ArgString_2023_09("echo should-not-run")], + ) + env = _wrap_env("wrap_env", probe) + step = _step_script_with_cancelation(None) + # Forward an invalid mode through the wrap round trip: the wrap + # action's own cancelation defers to a symbol that resolves to + # garbage at run time. + from openjd.model._format_strings import FormatString + from openjd.model.v2023_09 import CancelationMethodDeferred, ModelParsingContext + + ctx = ModelParsingContext(supported_extensions=["FEATURE_BUNDLE_1", "EXPR"]) + assert env.script is not None # _wrap_env always populates it + object.__setattr__( + env.script.actions.onWrapTaskRun, + "cancelation", + CancelationMethodDeferred( + mode=FormatString("{{WrappedStep.Name}}", context=ctx), + notifyPeriodInSeconds=None, + ), + ) + with Session(session_id=session_id, job_parameter_values={}) as session: + session.enter_environment(environment=env) + _run_until_ready(session) + session.run_task(step_script=step, task_parameter_values={}, step_name="NotAMode") + _run_until_ready(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert session.state == SessionState.READY_ENDING + + +# --------------------------------------------------------------------------- +# End-to-end delivery: canceling a running wrap hook must actually apply the +# method resolved at launch, not merely store it. +# --------------------------------------------------------------------------- + + +@serial_process +class TestWrapCancelationDelivery: + """The launch-time-resolution refactor removed cancel()'s ability to derive + the cancel method itself, so nothing but this test covers the wiring from + the stored method through to a delivered signal. Platform-gated like the + other signal-delivering cancel tests in this repo.""" + + @pytest.mark.skipif(not is_posix(), reason="Signals not yet implemented for non-posix") + def test_cancel_delivers_forwarded_notify_period(self, python_exe: str) -> None: + # GIVEN: a wrap hook whose own cancelation forwards the wrapped action's + # NOTIFY_THEN_TERMINATE mode and period, wrapping an action that declares + # a 1 second period. The hook runs a program that TRAPS SIGTERM, so only + # the terminate that follows the notify period can end it — a plain + # Terminate cancel would leave it running. + from datetime import timedelta + from pathlib import Path as _Path + + from openjd.model._format_strings import FormatString + from openjd.model.v2023_09 import CancelationMethodDeferred, ModelParsingContext + + from openjd.sessions._runner_base import NotifyCancelMethod + + ctx = ModelParsingContext(supported_extensions=["FEATURE_BUNDLE_1", "EXPR"]) + trapping_app = ( + _Path(__file__).parent / "support_files" / "app_20s_run_ignore_signal.py" + ).resolve() + hook = Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09(str(trapping_app))], + ) + object.__setattr__( + hook, + "cancelation", + CancelationMethodDeferred( + mode=FormatString("{{WrappedAction.Cancelation.Mode}}", context=ctx), + notifyPeriodInSeconds=FormatString( + "{{WrappedAction.Cancelation.NotifyPeriodInSeconds}}", context=ctx + ), + ), + ) + env = _wrap_env("wrap_env", hook) + step = _step_script_with_cancelation(_notify(1)) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=env) + _run_until_ready(session) + + session.run_task(step_script=step, task_parameter_values={}, step_name="Step") + deadline = time.time() + 10.0 + while ( + session.action_status is None or session.action_status.state != ActionState.RUNNING + ): + if time.time() > deadline: # pragma: no cover - timing guard + pytest.fail("wrap hook never reached RUNNING") + time.sleep(0.05) + + runner = session._runner + assert runner is not None + # The forwarded mode+period resolved at launch... + assert runner._resolved_cancel_method == NotifyCancelMethod( + terminate_delay=timedelta(seconds=1) + ) + + # WHEN + time.sleep(0.5) # let the subprocess install its signal handler + session.cancel_action() + + # THEN: ...and the notify-then-terminate method was really delivered, + # so a SIGTERM-trapping subprocess still ends, and well before its own + # 20 second runtime. + started = time.time() + _run_until_ready(session, timeout_s=15.0) + elapsed = time.time() - started + status = session.action_status + assert status is not None + assert status.state == ActionState.CANCELED + assert elapsed < 15.0 diff --git a/test/openjd/sessions_v0/test_wrap_scope_isolation.py b/test/openjd/sessions_v0/test_wrap_scope_isolation.py new file mode 100644 index 00000000..bd8e6f22 --- /dev/null +++ b/test/openjd/sessions_v0/test_wrap_scope_isolation.py @@ -0,0 +1,526 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""RFC 0008 two-scope isolation, in the inner -> hook direction. + +The wrap-actions design rests on two strictly separated scopes: the wrapped +action resolves against the INNER entity's own scope, and the hook resolves +against the WRAP environment's own scope plus the ``WrappedAction.*`` overlay. + +The wrap -> inner direction is covered by ``test_wrap_task_run.py`` and +``test_wrap_actions.py::TestWrapScopeSeparation``, and so is one part of this +direction: ``test_wrap_actions.py::test_inner_env_let_does_not_leak_into_hook_scope`` +already pins that an inner script's *script-level* ``let`` bindings stay out of a +hook's scope -- those live only in the copy ``_build_wrapped_inner_scope`` makes, +so they never leaked. + +What was NOT covered, and did leak, is everything the Session writes into the +inner entity's table directly, because the hook used to resolve against that same +table: + +- a wrapped task's ``Task.Param.*`` / ``Task.RawParam.*`` +- the running step's ``Step.Name`` +- the ``extra_let_bindings`` the *inner* environment was entered with + +``WrappedStep.Name`` exists in RFC 0008 precisely because ``Step.Name`` is not +meant to be reachable from a hook. openjd-model does not reject any of these +references in an environment script, so this runtime was the only gate. + +Every test here asserts on the symbol table the Session actually hands to the +hook's runner, captured at ``_make_env_script_runner``. Resolving a table the +test built itself would only re-assert the test's own construction. +""" + +from __future__ import annotations + +import time +import uuid +from pathlib import PurePosixPath +from typing import Any + +import pytest + +from openjd.model import ParameterValue, ParameterValueType, SymbolTable +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CommandString as CommandString_2023_09, + Environment as Environment_2023_09, + EnvironmentActions as EnvironmentActions_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, + StepActions as StepActions_2023_09, + StepScript as StepScript_2023_09, +) + +from openjd.sessions import ( + ActionState, + PathFormat, + PathMappingRule, + Session, + SessionState, +) + + +def _noop(python_exe: str) -> Action_2023_09: + """A do-nothing action that exists on every platform. + + ``true``/``echo`` are not native Windows executables, and every test in this + file requires its action to actually complete (see ``_run_until_ready``), so + the suite's ``python_exe`` fixture is used throughout rather than a shell + builtin. + """ + return Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09("pass")], + ) + + +def _wrap_env(python_exe: str, name: str = "WrapEnv") -> Environment_2023_09: + """A wrap environment whose three hooks are all no-ops. + + What the hooks print does not matter here: these tests inspect the symbol + table the Session builds for the hook, which keeps them independent of + whether a given symbol happens to be spellable in an ArgString under the + extensions the model was parsed with. Deliberately declares no script-level + ``let`` bindings, so a positive assertion about a binding can only be + attributable to ``_seed_wrap_env_scope``. + """ + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=_noop(python_exe), + onWrapTaskRun=_noop(python_exe), + onWrapEnvExit=_noop(python_exe), + ) + ), + ) + + +def _inner_env(python_exe: str, name: str = "Inner") -> Environment_2023_09: + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09(onEnter=_noop(python_exe), onExit=_noop(python_exe)) + ), + ) + + +def _step_script(python_exe: str) -> StepScript_2023_09: + return StepScript_2023_09(actions=StepActions_2023_09(onRun=_noop(python_exe))) + + +def _run_until_ready(session: Session, timeout_s: float = 10.0) -> None: + """Block until the session leaves RUNNING, then require that it got there. + + The trailing assertion is the load-bearing part. Twelve of the assertions in + this file are on a table that is fully built *before* the hook's subprocess + starts, so without it a hung or failed action would leave every one of them + asserting valid-but-unexercised state and passing. + """ + deadline = time.time() + timeout_s + while session.state == SessionState.RUNNING and time.time() < deadline: + time.sleep(0.05) + assert session.state != SessionState.RUNNING, ( + f"session did not leave RUNNING within {timeout_s}s; " + f"action_status={session.action_status}" + ) + + +class _HookScopeCapture: + """Records the symbol table the Session hands to each hook's runner. + + Patches ``_make_env_script_runner`` because its ``symtab`` argument *is* the + hook's resolution scope -- the runner resolves the hook's command, args, + timeout and cancelation against it, and evaluates the wrap env's own lets and + embedded files into it. Anything absent from it is unreachable from a hook. + + ``symtab`` is keyword-only on that method, so it always arrives in + ``kwargs``; if that ever changed, nothing would be recorded and + :meth:`table` would fail loudly rather than pass vacuously. + + Note that the captured tables are live objects: the runner mutates the one it + was given. Assertions therefore see the table as it was *used*, which is + stronger than as it was handed over. + """ + + def __init__(self, session: Session) -> None: + self.tables: list[SymbolTable] = [] + self._original = session._make_env_script_runner + + def _capturing(*args: Any, **kwargs: Any) -> Any: + symtab = kwargs.get("symtab") + if symtab is not None: + self.tables.append(symtab) + return self._original(*args, **kwargs) + + session._make_env_script_runner = _capturing # type: ignore[method-assign] + + def table(self, index: int, *, expected_count: int) -> SymbolTable: + """The ``index``-th captured scope, requiring exactly ``expected_count``. + + The count is asserted rather than just indexing from the end: a + regression that stopped invoking one of the hooks would otherwise leave a + test silently asserting against a previous hook's table. + """ + assert ( + len(self.tables) == expected_count + ), f"expected {expected_count} hook scope(s), captured {len(self.tables)}" + return self.tables[index] + + +def _defined(symtab: SymbolTable, name: str) -> bool: + """Is ``name`` resolvable in ``symtab``? + + ``SymbolTable.__contains__`` tests the backing table, which is exactly the + set of names both the legacy interpolation path and the EXPR engine resolve + from -- so this is "resolvable", not merely "present". + """ + return name in symtab + + +class TestTaskScopeDoesNotReachTheHook: + """onWrapTaskRun: the wrapped task's own symbols must not be in the hook's + scope.""" + + @pytest.mark.parametrize("leaked_symbol", ["Task.Param.Frame", "Task.RawParam.Frame"]) + def test_task_parameters_are_not_in_the_hook_scope( + self, leaked_symbol: str, python_exe: str + ) -> None: + # GIVEN: a wrap env active over a task that has parameters + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=_wrap_env(python_exe)) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN + session.run_task( + step_script=_step_script(python_exe), + task_parameter_values={ + "Frame": ParameterValue(type=ParameterValueType.INT, value="42") + }, + step_name="RenderStep", + ) + _run_until_ready(session) + + # THEN: the hook cannot see the wrapped task's parameters -- neither + # the value nor its EXPR type, which would disclose the parameter's + # existence on its own. + assert session.action_status is not None + assert session.action_status.state == ActionState.SUCCESS + hook_scope = capture.table(0, expected_count=1) + assert not _defined(hook_scope, leaked_symbol) + assert leaked_symbol not in hook_scope.expr_types + + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + def test_running_step_name_is_not_in_the_hook_scope(self, python_exe: str) -> None: + """A job-level wrap env has no step context of its own, so nothing + overwrites Step.Name -- which is how the running step's name used to + survive into the hook.""" + # GIVEN: a wrap env entered with NO step_name + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=_wrap_env(python_exe)) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN: a task of a named step runs under it + session.run_task( + step_script=_step_script(python_exe), + task_parameter_values={}, + step_name="RenderStep", + ) + _run_until_ready(session) + + # THEN + hook_scope = capture.table(0, expected_count=1) + assert not _defined(hook_scope, "Step.Name") + # ...and the hook still gets the name through the channel RFC 0008 + # provides for it. + assert hook_scope["WrappedStep.Name"] == "RenderStep" + + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + def test_wrap_envs_own_step_name_still_reaches_the_hook(self, python_exe: str) -> None: + """The fix must not cost the wrap env its own enter-time step context.""" + # GIVEN: a wrap env entered as part of "StepA" + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment( + environment=_wrap_env(python_exe), step_name="StepA" + ) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN: a task of a DIFFERENT step runs under it + session.run_task( + step_script=_step_script(python_exe), + task_parameter_values={}, + step_name="StepB", + ) + _run_until_ready(session) + + # THEN: the hook sees its OWN step, not the running one. + hook_scope = capture.table(0, expected_count=1) + assert hook_scope["Step.Name"] == "StepA" + assert hook_scope["WrappedStep.Name"] == "StepB" + + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + def test_session_scope_still_reaches_the_hook(self, python_exe: str) -> None: + """Job parameters and session symbols are legitimately session scope and + must survive the rebuild -- values AND their EXPR types.""" + # GIVEN + with Session( + session_id=uuid.uuid4().hex, + job_parameter_values={ + "JobParam": ParameterValue(type=ParameterValueType.STRING, value="jp") + }, + job_name="MyJob", + ) as session: + wrap_id = session.enter_environment(environment=_wrap_env(python_exe)) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN + session.run_task( + step_script=_step_script(python_exe), + task_parameter_values={}, + step_name="S", + ) + _run_until_ready(session) + + # THEN + hook_scope = capture.table(0, expected_count=1) + assert hook_scope["Param.JobParam"] == "jp" + assert hook_scope["RawParam.JobParam"] == "jp" + assert hook_scope["Job.Name"] == "MyJob" + assert hook_scope["Session.WorkingDirectory"] == str(session.working_directory) + # EXPR typing is part of the scope: without it a PATH-typed symbol + # silently degrades to a plain string inside a hook's expressions. + assert hook_scope.expr_types["Session.WorkingDirectory"] == ( + ParameterValueType.PATH.value + ) + assert hook_scope.expr_types["Param.JobParam"] == (ParameterValueType.STRING.value) + + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + def test_path_mapping_reaches_the_hook_intact(self, python_exe: str) -> None: + """A hook resolving Session.PathMappingRulesFile, or calling the EXPR + host function apply_path_mapping(), must see this session's real rules. + + Both halves matter and neither was covered: the rules FILE (a hook that + reads it) and the engine's host RULES (a hook that calls + apply_path_mapping). With the host rules absent the function silently + becomes the identity -- no error, wrong paths. + """ + # GIVEN: a session that actually has path mapping rules + rules = [ + PathMappingRule( + source_path_format=PathFormat.POSIX, + source_path=PurePosixPath("/mnt/src"), + destination_path=PurePosixPath("/mnt/dst"), + ) + ] + with Session( + session_id=uuid.uuid4().hex, + job_parameter_values={}, + path_mapping_rules=rules, + ) as session: + wrap_id = session.enter_environment(environment=_wrap_env(python_exe)) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN + session.run_task( + step_script=_step_script(python_exe), + task_parameter_values={}, + step_name="S", + ) + _run_until_ready(session) + + # THEN + hook_scope = capture.table(0, expected_count=1) + assert hook_scope["Session.HasPathMappingRules"] == "true" + assert hook_scope.expr_types["Session.HasPathMappingRules"] == ( + ParameterValueType.BOOL.value + ) + assert str(hook_scope["Session.PathMappingRulesFile"]).endswith(".json") + # The engine's host context must carry the rules, not an empty list. + assert hook_scope.expr_host_rules, "a hook lost the session's path mapping rules" + assert len(hook_scope.expr_host_rules) == len(rules) + assert hook_scope.expr_host_rules == session._expr_host_rules + + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + +class TestInnerEnvironmentScopeDoesNotReachTheHook: + """onWrapEnvEnter / onWrapEnvExit: the INNER environment's enter-time context + must not be in the hook's scope.""" + + @pytest.mark.parametrize("phase", ["enter", "exit"]) + def test_inner_extra_let_bindings_are_not_in_the_hook_scope( + self, phase: str, python_exe: str + ) -> None: + # GIVEN: a wrap env, and an inner env entered with its own step-level + # bindings and step name + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=_wrap_env(python_exe)) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN + inner_id = session.enter_environment( + environment=_inner_env(python_exe), + extra_let_bindings=["inner_secret = 'INNER-ONLY'"], + step_name="InnerStep", + ) + _run_until_ready(session) + if phase == "exit": + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + + # THEN: neither the inner env's binding nor its step name is + # reachable from the hook that intercepted it. Index the hook we + # care about rather than the most recent capture, so a hook that + # stopped running cannot pass by inheriting the other's table. + expected = 2 if phase == "exit" else 1 + hook_scope = capture.table(expected - 1, expected_count=expected) + assert not _defined(hook_scope, "inner_secret") + assert not _defined(hook_scope, "Step.Name") + assert hook_scope["WrappedEnv.Name"] == "Inner" + + if phase == "enter": + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + @pytest.mark.parametrize("phase", ["enter", "exit"]) + def test_wrap_envs_own_bindings_still_reach_the_hook(self, phase: str, python_exe: str) -> None: + """Guard the behaviour `_seed_wrap_env_scope` exists for. + + ``test_wrap_task_run.py::test_env_hooks_resolve_step_level_let_bindings`` + covers the same ground by asserting the hook merely SUCCEEDs; this + asserts the binding is in the hook's scope with the right value, which is + what distinguishes "seeded" from "the hook happened not to need it". + """ + # GIVEN: a wrap env entered WITH step-level bindings + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment( + environment=_wrap_env(python_exe), + extra_let_bindings=["wrap_secret = 'WRAP-OWN'"], + step_name="WrapStep", + ) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN + inner_id = session.enter_environment(environment=_inner_env(python_exe)) + _run_until_ready(session) + if phase == "exit": + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + + # THEN. `let` bindings are stored as the EXPR engine's typed value, + # so compare the rendered form rather than the object. + expected = 2 if phase == "exit" else 1 + hook_scope = capture.table(expected - 1, expected_count=expected) + assert str(hook_scope["wrap_secret"]) == "WRAP-OWN" + assert hook_scope["Step.Name"] == "WrapStep" + + if phase == "enter": + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + +class TestWrappedActionStillResolvesInTheInnerScope: + """The other direction must be unaffected: the wrapped action still resolves + against the inner entity's own scope, which is what makes the split useful.""" + + def test_wrapped_args_resolve_the_running_tasks_parameters(self, python_exe: str) -> None: + # GIVEN: a step whose onRun references its own task parameter + step = StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("frame-{{Task.Param.Frame}}")], + ) + ) + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=_wrap_env(python_exe)) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN + session.run_task( + step_script=step, + task_parameter_values={ + "Frame": ParameterValue(type=ParameterValueType.INT, value="42") + }, + step_name="RenderStep", + ) + _run_until_ready(session) + + # THEN: the wrapped action resolved with the task's own parameter, + # even though the hook's scope cannot see it. + hook_scope = capture.table(0, expected_count=1) + assert hook_scope["WrappedAction.Args"] == ["frame-42"] + assert not _defined(hook_scope, "Task.Param.Frame") + + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + def test_two_tasks_do_not_bleed_into_each_other(self, python_exe: str) -> None: + """The hook's scope is rebuilt per action, so one task's parameters must + not survive into the next task's hook.""" + # GIVEN + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=_wrap_env(python_exe)) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN: two tasks run under the same wrap env + for value in ("1", "2"): + session.run_task( + step_script=_step_script(python_exe), + task_parameter_values={ + "Frame": ParameterValue(type=ParameterValueType.INT, value=value) + }, + step_name="RenderStep", + ) + _run_until_ready(session) + + # THEN: distinct tables, neither carrying the task's parameters. + first = capture.table(0, expected_count=2) + second = capture.table(1, expected_count=2) + assert first is not second + for scope in (first, second): + assert not _defined(scope, "Task.Param.Frame") + + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + +class TestHookScopeBuilderUnit: + """Direct unit coverage of the builder for the one case the end-to-end tests + cannot reach: a table that has not been through path mapping.""" + + def test_missing_path_mapping_symbols_are_tolerated(self, python_exe: str) -> None: + """Every production caller materializes path mapping first, so the + membership guard in the builder has no end-to-end trigger. Pin it here + rather than leave an unexercised branch.""" + # GIVEN a table with no path-mapping symbols at all + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + revision = _step_script(python_exe).revision + + # WHEN / THEN: no KeyError + hook_symtab = session._build_wrap_hook_scope(revision, SymbolTable()) + assert not _defined(hook_symtab, "Session.PathMappingRulesFile") + assert not _defined(hook_symtab, "Session.HasPathMappingRules") diff --git a/test/openjd/sessions_v0/test_wrap_task_run.py b/test/openjd/sessions_v0/test_wrap_task_run.py new file mode 100644 index 00000000..53446ebc --- /dev/null +++ b/test/openjd/sessions_v0/test_wrap_task_run.py @@ -0,0 +1,841 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for the ``onWrapTaskRun`` environment action (RFC 0008). + +These tests exercise the end-to-end behaviour that a job template's ``onRun`` is +intercepted and the active environment's ``onWrapTaskRun`` is executed in its +place, with ``WrappedAction.Command``, ``WrappedAction.Args``, +``WrappedAction.Environment``, ``WrappedAction.Timeout``, and +``WrappedStep.Name`` injected into the wrap action's symbol table. +""" + +from __future__ import annotations + +import time +import uuid + +import pytest + +from openjd.model import SymbolTable +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CommandString as CommandString_2023_09, + Environment as Environment_2023_09, + EnvironmentActions as EnvironmentActions_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, + StepActions as StepActions_2023_09, + StepScript as StepScript_2023_09, +) +from openjd.sessions import ActionState, ActionStatus, Session, SessionState + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +_NOOP = Action_2023_09(command=CommandString_2023_09("true")) + + +def _wrap_env(name: str, wrap_action: Action_2023_09) -> Environment_2023_09: + """Build an Environment with ``onWrapTaskRun`` set to ``wrap_action`` + and the other two wrap hooks set to no-ops (all-or-nothing rule).""" + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=_NOOP, + onWrapTaskRun=wrap_action, + onWrapEnvExit=_NOOP, + ), + ), + ) + + +def _step_script(command: str, args: list[str]) -> StepScript_2023_09: + """Build a minimal StepScript that runs ``command`` with ``args``.""" + return StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09(command), + args=[ArgString_2023_09(a) for a in args] if args else None, + ) + ) + ) + + +def _run_until_ready(session: Session, timeout_s: float = 10.0) -> None: + """Block until the session transitions back to READY or the timeout elapses.""" + deadline = time.time() + timeout_s + while session.state == SessionState.RUNNING and time.time() < deadline: + time.sleep(0.05) + + +# --------------------------------------------------------------------------- +# Unit tests on the pure helpers — no subprocess needed +# --------------------------------------------------------------------------- + + +class TestInjectWrappedTaskSymbols: + """Unit tests for the ``_inject_wrapped_task_symbols`` helper.""" + + def test_injects_wrapped_command_and_args_as_list(self) -> None: + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + script = _step_script("python3", ["-c", "print('hi')"]) + session._inject_wrapped_task_symbols(symtab, script, "MyStep", inner_symtab=symtab) + + assert symtab["WrappedAction.Command"] == "python3" + assert symtab["WrappedAction.Args"] == ["-c", "print('hi')"] + assert isinstance(symtab["WrappedAction.Args"], list) + assert symtab["WrappedStep.Name"] == "MyStep" + finally: + session.cleanup() + + def test_injects_empty_args_when_step_has_no_args(self) -> None: + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + script = _step_script("whoami", []) + session._inject_wrapped_task_symbols(symtab, script, "Step1", inner_symtab=symtab) + + assert symtab["WrappedAction.Command"] == "whoami" + assert symtab["WrappedAction.Args"] == [] + finally: + session.cleanup() + + def test_injects_typed_args_null_skip_and_list_flatten(self) -> None: + # RFC 0005 §1.3.2 typed argument semantics (openjd-rs parity: the + # wrapped path in seed_wrapped_action_symbols resolves through the + # same resolve_action_args as the runner): a whole-field list + # expression flattens inline (one argument per element), a + # whole-field null is skipped, and the hook sees exactly the argv + # the wrapped action would have run with unwrapped. + from openjd.model.v2023_09 import ModelParsingContext + from openjd.sessions._runner_base import resolve_action_arg_values + + context = ModelParsingContext(supported_extensions=["EXPR"]) + script = StepScript_2023_09.model_validate( + { + "actions": { + "onRun": { + "command": "echo", + "args": ["front", '{{ ["a", "b c"] }}', "{{ null }}", "back"], + } + } + }, + context=context, + ) + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + session._inject_wrapped_task_symbols(symtab, script, "MyStep", inner_symtab=symtab) + assert symtab["WrappedAction.Args"] == ["front", "a", "b c", "back"] + # The unwrapped enforcement path (_run_action) resolves the same + # action's args via the same shared helper — wrapped and + # unwrapped runs of this action use identical argv. + assert resolve_action_arg_values(script.actions.onRun.args, symtab) == [ + "front", + "a", + "b c", + "back", + ] + finally: + session.cleanup() + + def test_injects_wrapped_environment_as_key_value_list(self) -> None: + # RFC 0008 (openjd-rs #277): WrappedAction.Environment carries every + # session-defined variable — openjd_env definitions (applied via + # simplify_ordered_changes) AND the declarative variables:-map seed. + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + from openjd.sessions._session import ( + EnvironmentVariableSetChange, + SimplifiedEnvironmentVariableChanges, + ) + + fake_id = "env-1" + session._environments_entered.append(fake_id) + changes = SimplifiedEnvironmentVariableChanges({"DECLARED": "from-variables-map"}) + changes.simplify_ordered_changes( + [ + EnvironmentVariableSetChange(name="FOO", value="bar"), + EnvironmentVariableSetChange(name="BAZ", value="qux"), + ] + ) + session._created_env_vars[fake_id] = changes + + symtab = SymbolTable() + script = _step_script("echo", ["hi"]) + session._inject_wrapped_task_symbols(symtab, script, "Step1", inner_symtab=symtab) + + task_env = symtab["WrappedAction.Environment"] + assert isinstance(task_env, list) + normalized = {item.upper() for item in task_env} + # Both openjd_env-set variables and the variables:-map seed are + # surfaced (openjd-rs #277). + assert normalized == { + "DECLARED=from-variables-map".upper(), + "FOO=bar".upper(), + "BAZ=qux".upper(), + } + finally: + session.cleanup() + + def test_injects_timeout_none_when_no_timeout(self) -> None: + # WrappedAction.Timeout is int? (RFC 0008): None (null) when the + # wrapped action specifies no timeout, so whole-field forwarding + # (timeout: "{{WrappedAction.Timeout}}") drops the field. + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + script = _step_script("echo", ["hi"]) + session._inject_wrapped_task_symbols(symtab, script, "Step1", inner_symtab=symtab) + + assert symtab["WrappedAction.Timeout"] is None + finally: + session.cleanup() + + def test_find_wrap_environment_returns_none_when_no_env_has_wrap(self) -> None: + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + assert session._find_wrap_environment(hook="onWrapTaskRun") is None + finally: + session.cleanup() + + +# --------------------------------------------------------------------------- +# Integration tests — actually run a subprocess wrap action +# --------------------------------------------------------------------------- + + +class TestWrapTaskRunExecution: + """Integration tests that enter an environment with ``onWrapTaskRun`` and + run a task through it, verifying the wrap action is what actually ran.""" + + def test_no_wrap_runs_original_step(self, caplog: pytest.LogCaptureFixture) -> None: + # GIVEN: a session with no environment defining onWrapTaskRun. + session_id = uuid.uuid4().hex + step = _step_script("echo", ["original-step"]) + + # WHEN: we run the task. + with Session(session_id=session_id, job_parameter_values={}) as session: + session.run_task(step_script=step, task_parameter_values={}, step_name="Step") + _run_until_ready(session) + + # THEN: the original step ran, not a wrap action. + assert session.state == SessionState.READY + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + assert any("original-step" in msg for msg in caplog.messages) + + def test_wrap_action_intercepts_step(self, caplog: pytest.LogCaptureFixture) -> None: + # GIVEN: an environment that defines onWrapTaskRun. + session_id = uuid.uuid4().hex + wrap = Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ArgString_2023_09("-c"), ArgString_2023_09("echo WRAPPED-RAN")], + ) + env = _wrap_env("container_env", wrap) + + step = _step_script("echo", ["original-step"]) + + with Session(session_id=session_id, job_parameter_values={}) as session: + # Enter the wrap env (no onEnter, nothing runs but the env becomes active). + session.enter_environment(environment=env) + _run_until_ready(session) + + # WHEN: we run a task. + session.run_task(step_script=step, task_parameter_values={}, step_name="Step") + _run_until_ready(session) + + # THEN: the wrap action ran, not the original step. + assert session.state == SessionState.READY + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + messages = "\n".join(caplog.messages) + assert "WRAPPED-RAN" in messages + assert "original-step" not in messages + + def test_wrap_action_receives_wrapped_command_symbol( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # GIVEN: a wrap action that echoes WrappedAction.Command via a format string. + session_id = uuid.uuid4().hex + wrap = Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09("echo CMD={{WrappedAction.Command}}"), + ], + ) + env = _wrap_env("container_env", wrap) + step = _step_script("maya-batch", ["-render", "scene.ma"]) + + with Session(session_id=session_id, job_parameter_values={}) as session: + session.enter_environment(environment=env) + _run_until_ready(session) + + session.run_task(step_script=step, task_parameter_values={}, step_name="Step") + _run_until_ready(session) + + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + messages = "\n".join(caplog.messages) + assert "CMD=maya-batch" in messages + + def test_innermost_environment_wins(self, caplog: pytest.LogCaptureFixture) -> None: + # GIVEN: two environments — only the inner one defines onWrapTaskRun. + session_id = uuid.uuid4().hex + outer = Environment_2023_09( + name="outer", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09("echo outer-enter"), + ], + ) + ) + ), + ) + inner = _wrap_env( + "inner", + Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ArgString_2023_09("-c"), ArgString_2023_09("echo INNER-WRAP")], + ), + ) + step = _step_script("echo", ["original"]) + + with Session(session_id=session_id, job_parameter_values={}) as session: + session.enter_environment(environment=outer) + _run_until_ready(session) + session.enter_environment(environment=inner) + _run_until_ready(session) + + session.run_task(step_script=step, task_parameter_values={}, step_name="Step") + _run_until_ready(session) + + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + messages = "\n".join(caplog.messages) + assert "INNER-WRAP" in messages + assert "original" not in messages + + def test_wrap_action_runs_multiple_tasks(self, caplog: pytest.LogCaptureFixture) -> None: + # Regression: symbols are re-injected cleanly per task. + session_id = uuid.uuid4().hex + wrap = Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09("echo RAN={{WrappedAction.Command}}"), + ], + ) + env = _wrap_env("env", wrap) + + with Session(session_id=session_id, job_parameter_values={}) as session: + session.enter_environment(environment=env) + _run_until_ready(session) + + session.run_task( + step_script=_step_script("cmd-one", []), + task_parameter_values={}, + step_name="Step", + ) + _run_until_ready(session) + session.run_task( + step_script=_step_script("cmd-two", []), + task_parameter_values={}, + step_name="Step", + ) + _run_until_ready(session) + + messages = "\n".join(caplog.messages) + assert "RAN=cmd-one" in messages + assert "RAN=cmd-two" in messages + + +# --------------------------------------------------------------------------- +# Security and execution constraints — RFC 0008 test matrix +# --------------------------------------------------------------------------- +# +# These tests cover the 8 scenarios the RFC calls out under "Recommended test +# cases for implementation". They verify that the symbol injection layer +# preserves WrappedAction.Command and WrappedAction.Args byte-for-byte — no +# shell expansion, no quoting transformation, no truncation. + + +class TestSecurityAndExecutionConstraints: + """RFC 0008 §"Recommended test cases for implementation" matrix.""" + + def _inject(self, command: str, args: list[str]) -> tuple[str, list[str]]: + """Helper: run the injection and return the resolved WrappedAction.Command/Args.""" + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + script = _step_script(command, args) + session._inject_wrapped_task_symbols(symtab, script, "TestStep", inner_symtab=symtab) + return symtab["WrappedAction.Command"], symtab["WrappedAction.Args"] + finally: + session.cleanup() + + # --- 1. Nested quoting ------------------------------------------------ + + def test_nested_quoting_preserved(self) -> None: + # echo "O'Reilly's Guide" — both " and ' must survive injection. + cmd, args = self._inject("echo", ["O'Reilly's Guide"]) + assert cmd == "echo" + assert args == ["O'Reilly's Guide"] + + def test_double_quotes_preserved(self) -> None: + cmd, args = self._inject("python3", ["-c", 'print("hello world")']) + assert cmd == "python3" + assert args == ["-c", 'print("hello world")'] + + # --- 2. Shell metacharacters ------------------------------------------ + + def test_shell_metacharacters_preserved(self) -> None: + # `, $, |, &&, ;, >, <, *, ?, (, ), \, all in one arg. + nasty = "$(cat /etc/passwd); `id`; rm -rf / || echo pwned" + cmd, args = self._inject("echo", [nasty, "a|b", "c&&d", "e;f"]) + assert cmd == "echo" + assert args == [nasty, "a|b", "c&&d", "e;f"] + + def test_backticks_preserved(self) -> None: + cmd, args = self._inject("echo", ["`whoami`", "$USER"]) + assert args == ["`whoami`", "$USER"] + # The $ must NOT have been expanded by the runtime. + assert "$USER" in args + + # --- 3. Path traversal ------------------------------------------------ + + def test_path_traversal_preserved_literally(self) -> None: + # The runtime must not resolve or reject `../` paths — the wrap + # script + container boundary is what prevents escape. + cmd, args = self._inject("cat", ["../../../etc/passwd", "/tmp/../etc/shadow"]) + assert cmd == "cat" + assert args == ["../../../etc/passwd", "/tmp/../etc/shadow"] + + # --- 4. Shell globbing ------------------------------------------------ + + def test_glob_characters_preserved(self) -> None: + # ls *.txt must be passed literally — no expansion at injection time. + cmd, args = self._inject("ls", ["*.txt", "?.log", "[abc]*"]) + assert cmd == "ls" + assert args == ["*.txt", "?.log", "[abc]*"] + + # --- 5. Unicode paths ------------------------------------------------- + + def test_unicode_cjk_preserved(self) -> None: + path = "/projects/映画/シーン01/レンダー.exr" + cmd, args = self._inject("render", ["--scene", path]) + assert cmd == "render" + assert args == ["--scene", path] + assert len(path.encode("utf-8")) > len(path) + + def test_unicode_emoji_preserved(self) -> None: + cmd, args = self._inject("echo", ["🎬 render 🎥", "🔥"]) + assert args == ["🎬 render 🎥", "🔥"] + + def test_unicode_mixed_scripts_preserved(self) -> None: + cmd, args = self._inject( + "tool", + ["--русский", "--中文", "--日本語", "--한국어", "--العربية"], + ) + assert args == ["--русский", "--中文", "--日本語", "--한국어", "--العربية"] + + # --- 6. Empty and whitespace-only arguments --------------------------- + + def test_empty_string_arg_preserved(self) -> None: + cmd, args = self._inject("echo", ["before", "", "after"]) + assert cmd == "echo" + assert args == ["before", "", "after"] + assert len(args) == 3 + + def test_whitespace_only_arg_preserved(self) -> None: + cmd, args = self._inject("echo", [" ", " ", " "]) + assert args == [" ", " ", " "] + + # --- 7. Newlines in arguments ----------------------------------------- + + def test_newline_in_arg_preserved(self) -> None: + # The EXPR extension relaxes the ArgString regex to accept + # control characters; the runtime must preserve them literally. + cmd, args = self._inject("echo", ["line1\nline2"]) + assert cmd == "echo" + assert args == ["line1\nline2"] + + # --- 8. Near-limit command length ------------------------------------- + + def test_large_argument_preserved(self) -> None: + big = "x" * (100 * 1024) + cmd, args = self._inject("cat", ["--data", big]) + assert cmd == "cat" + assert args == ["--data", big] + assert len(args[1]) == 100 * 1024 + + def test_many_arguments_preserved(self) -> None: + many = [f"arg{i}" for i in range(1000)] + cmd, args = self._inject("tool", many) + assert cmd == "tool" + assert args == many + assert len(args) == 1000 + + +# --------------------------------------------------------------------------- +# Wrap-environment embedded-file reuse across tasks. +# +# The wrap environment's embedded-file PATHS are allocated once and reused +# for every task run through the wrap (so Env.File.* symbols stay stable and +# unnamed files do not accumulate on disk), while the file CONTENTS are +# re-resolved and rewritten per task (data may reference WrappedAction.*). +# --------------------------------------------------------------------------- + + +class TestWrapEnvEmbeddedFileReuse: + def _wrap_env_with_unnamed_file(self) -> Environment_2023_09: + from openjd.model.v2023_09 import ( + DataString as DataString_2023_09, + EmbeddedFileText as EmbeddedFileText_2023_09, + EmbeddedFileTypes as EmbeddedFileTypes_2023_09, + ) + + # The embedded file is UNNAMED (no `filename`), so its on-disk path + # is mkstemp-allocated; its data references WrappedAction.Command, + # so contents must be rewritten per task. + return Environment_2023_09( + name="WrapEnv", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=_NOOP, + onWrapTaskRun=Action_2023_09( + command=CommandString_2023_09("cat"), + args=[ArgString_2023_09("{{ Env.File.WrapData }}")], + ), + onWrapEnvExit=_NOOP, + ), + embeddedFiles=[ + EmbeddedFileText_2023_09( + name="WrapData", + type=EmbeddedFileTypes_2023_09.TEXT, + data=DataString_2023_09("wrapped-command={{WrappedAction.Command}}\n"), + ) + ], + ), + ) + + def test_unnamed_wrap_file_path_reused_across_tasks(self) -> None: + # GIVEN: a wrap environment with an unnamed embedded file whose data + # references WrappedAction.Command, and three tasks with distinct + # wrapped commands (never executed; the wrap action runs instead). + env = self._wrap_env_with_unnamed_file() + commands = ("cmd-one", "cmd-two", "cmd-three") + file_paths = [] + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + identifier = session.enter_environment(environment=env) + _run_until_ready(session) + assert session.state == SessionState.READY + + for command in commands: + # WHEN: a task runs through the wrap. + session.run_task( + step_script=_step_script(command, []), + task_parameter_values={}, + step_name="Step", + ) + _run_until_ready(session) + + # THEN: the task succeeded, and the files directory contains + # exactly ONE file for the record (not one per task) ... + assert session.state == SessionState.READY + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + files = [p for p in session.files_directory.iterdir() if p.is_file()] + assert len(files) == 1 + # ... whose contents reflect THIS task's wrapped command. + assert files[0].read_text() == f"wrapped-command={command}\n" + file_paths.append(files[0]) + + # THEN: the file path is identical across all three tasks. + assert file_paths[0] == file_paths[1] == file_paths[2] + + session.exit_environment(identifier=identifier) + _run_until_ready(session) + + +# --------------------------------------------------------------------------- +# step_name is required once a wrap environment is active: RFC 0008's +# WrappedStep.Name has no value to render without it, and has a +# minimum length of one, so there is no empty sentinel to fall back on. +# --------------------------------------------------------------------------- + + +class TestWrappedStepNameRequired: + def test_run_task_without_step_name_raises(self) -> None: + # GIVEN: a wrap environment is entered + env = _wrap_env( + "container", + Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("{{WrappedStep.Name}}")], + ), + ) + step = _step_script("echo", ["hello"]) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + identifier = session.enter_environment(environment=env) + _run_until_ready(session) + + # WHEN / THEN: omitting step_name is caller misuse, reported as such + # rather than silently rendering an empty container name. + with pytest.raises(ValueError, match="requires step_name"): + session.run_task(step_script=step, task_parameter_values={}) + + # AND: the session is left usable — nothing was started, so the + # caller can retry with a step name. + assert session.state == SessionState.READY + session.run_task(step_script=step, task_parameter_values={}, step_name="RetriedStep") + _run_until_ready(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + + session.exit_environment(identifier=identifier) + _run_until_ready(session) + + def test_run_task_without_step_name_is_fine_unwrapped(self) -> None: + # GIVEN: no wrap environment + step = _step_script("echo", ["hello"]) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + # WHEN: step_name is still optional on the unwrapped path + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + # THEN + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + + +# --------------------------------------------------------------------------- +# A wrap hook resolves in the wrap environment's own scope, which in openjd-rs +# is that environment's frozen enter-time symbol table. So a step environment +# that defines wrap hooks must carry the step-level `let` bindings it was +# entered with into every hook invocation -- without letting them replace the +# wrapped action's own resolution scope. +# --------------------------------------------------------------------------- + + +class TestWrapHookSeesEnterTimeStepScope: + def test_hook_resolves_step_level_let_bindings(self, tmp_path) -> None: + # GIVEN: a wrap environment entered with a step's step-level bindings, + # and a hook that references one of them. + env = _wrap_env( + "WrapEnv", + Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("HOOK-{{greeting}}")], + ), + ) + step = _step_script("echo", ["INNER"]) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + identifier = session.enter_environment( + environment=env, + extra_let_bindings=["greeting = 'from-step-let'"], + step_name="Step1", + ) + _run_until_ready(session) + assert session.action_status is not None + assert session.action_status.state == ActionState.SUCCESS + + # WHEN + session.run_task(step_script=step, task_parameter_values={}, step_name="Step1") + _run_until_ready(session) + + # THEN: the hook resolved the step-level binding instead of failing + # with "Undefined variable". + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + + session.exit_environment(identifier=identifier) + _run_until_ready(session) + + def test_wrap_env_step_name_does_not_reach_the_wrapped_action(self) -> None: + """The wrap env's enter-time step context must not replace the running + step's in the wrapped action's own scope. + + This goes through ``run_task`` on purpose: the ordering it pins lives in + ``run_task`` (seed the hook's scope only *after* the wrapped action's own + scope has been built), so a test that calls the two helpers itself in its + own order would re-assert its own script instead. + """ + # GIVEN: a wrap env entered for "StepA", and a step whose onRun echoes + # {{Step.Name}} -- which must resolve to the step actually running. + hook_args: list[str] = [] + + env = _wrap_env( + "WrapEnv", + Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("{{Step.Name}}")], + ), + ) + step = _step_script("echo", ["{{Step.Name}}"]) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + identifier = session.enter_environment( + environment=env, extra_let_bindings=None, step_name="StepA" + ) + _run_until_ready(session) + + original = session._inject_wrapped_task_symbols + + def _capture(symtab, step_script, step_name, *, inner_symtab): + original(symtab, step_script, step_name, inner_symtab=inner_symtab) + hook_args.extend(symtab["WrappedAction.Args"]) + + session._inject_wrapped_task_symbols = _capture # type: ignore[method-assign] + + # WHEN: a task of a DIFFERENT step runs under that wrap env + session.run_task(step_script=step, task_parameter_values={}, step_name="StepB") + _run_until_ready(session) + + # THEN: the wrapped action resolved with the RUNNING step's name. + # Seeding the hook's scope before injection instead would give StepA. + assert hook_args == ["StepB"] + assert session.action_status is not None + assert session.action_status.state == ActionState.SUCCESS + + session.exit_environment(identifier=identifier) + _run_until_ready(session) + + @pytest.mark.parametrize("hook_phase", ["enter", "exit"]) + def test_env_hooks_resolve_step_level_let_bindings(self, hook_phase: str) -> None: + """The seeding applies to onWrapEnvEnter and onWrapEnvExit too, not just + onWrapTaskRun.""" + # GIVEN: a wrap env entered with a step-level binding, and an inner env + # whose onEnter/onExit the hooks intercept. + wrap_action = Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("HOOK-{{greeting}}")], + ) + env = Environment_2023_09( + name="WrapEnv", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=wrap_action, + onWrapTaskRun=_NOOP, + onWrapEnvExit=wrap_action, + ), + ), + ) + inner = Environment_2023_09( + name="Inner", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09(command=CommandString_2023_09("true")), + onExit=Action_2023_09(command=CommandString_2023_09("true")), + ), + ), + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment( + environment=env, + extra_let_bindings=["greeting = 'from-step-let'"], + step_name="Step1", + ) + _run_until_ready(session) + + # WHEN + inner_id = session.enter_environment(environment=inner) + _run_until_ready(session) + if hook_phase == "enter": + # THEN: the onWrapEnvEnter hook resolved the binding + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + else: + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + # THEN: the onWrapEnvExit hook resolved the binding + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + + if hook_phase == "enter": + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + +# --------------------------------------------------------------------------- +# The session must never claim RUNNING while it has no runner. cancel_action() +# is a cross-thread API guarded only by `state == RUNNING`, so such a window is +# one in which a cancel is lost -- and the RFC 0008 branches do real work +# (materializing the inner entity's embedded files, evaluating its `let` +# bindings, allocating the hook's file records) before a runner exists. +# --------------------------------------------------------------------------- + + +class TestNoRunningWithoutRunner: + def _wrap_env_all_hooks(self) -> Environment_2023_09: + act = Action_2023_09(command=CommandString_2023_09("true")) + return Environment_2023_09( + name="WrapEnv", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=act, onWrapTaskRun=act, onWrapEnvExit=act + ), + ), + ) + + def _inner_env(self) -> Environment_2023_09: + act = Action_2023_09(command=CommandString_2023_09("true")) + return Environment_2023_09( + name="Inner", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09(onEnter=act, onExit=act), + ), + ) + + def test_state_is_not_running_during_wrap_setup(self) -> None: + """Observed at the last point before the runner is built, in all three + wrap paths.""" + observed: list[tuple[str, SessionState]] = [] + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + original = session._get_wrap_env_file_records + + def _observe(wrap_env): # type: ignore[no-untyped-def] + # _get_wrap_env_file_records is the last step before the runner + # is constructed, so this is the worst case for the window. + observed.append((wrap_env.name, session.state)) + return original(wrap_env) + + session._get_wrap_env_file_records = _observe # type: ignore[method-assign] + + wrap_id = session.enter_environment(environment=self._wrap_env_all_hooks()) + _run_until_ready(session) + + inner_id = session.enter_environment(environment=self._inner_env()) + _run_until_ready(session) + + session.run_task( + step_script=_step_script("echo", ["hi"]), + task_parameter_values={}, + step_name="Step1", + ) + _run_until_ready(session) + + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + # THEN: all three wrap paths were exercised, and none of them had + # already flipped the session to RUNNING. + assert len(observed) == 3, observed + assert all(state is not SessionState.RUNNING for _, state in observed), observed