From 498fcd3f00f2cda3a1f4be28e741887e2e6f96e8 Mon Sep 17 00:00:00 2001 From: primorLee <252785625+primorLee@users.noreply.github.com> Date: Sat, 22 Aug 2026 04:11:10 +0800 Subject: [PATCH 1/2] fix(terminal): lower agent process priority Co-authored-by: openhands --- .../terminal/terminal/process_priority.py | 13 ++++ .../terminal/terminal/subprocess_terminal.py | 5 +- .../tools/terminal/terminal/tmux_pane_pool.py | 12 +++- .../tools/terminal/terminal/tmux_terminal.py | 10 ++- .../terminal/terminal/windows_terminal.py | 1 + tests/tools/terminal/test_pool_integration.py | 16 +++++ tests/tools/terminal/test_process_priority.py | 63 +++++++++++++++++++ tests/tools/terminal/test_windows_terminal.py | 15 +++++ 8 files changed, 128 insertions(+), 7 deletions(-) create mode 100644 openhands-tools/openhands/tools/terminal/terminal/process_priority.py create mode 100644 tests/tools/terminal/test_process_priority.py diff --git a/openhands-tools/openhands/tools/terminal/terminal/process_priority.py b/openhands-tools/openhands/tools/terminal/terminal/process_priority.py new file mode 100644 index 0000000000..0abfcae649 --- /dev/null +++ b/openhands-tools/openhands/tools/terminal/terminal/process_priority.py @@ -0,0 +1,13 @@ +"""Platform-specific process priority policy for agent terminals.""" + +import platform + + +def get_process_priority_prefix() -> tuple[str, ...]: + """Return an argv prefix that lowers an agent terminal's CPU priority.""" + system = platform.system() + if system == "Darwin": + return ("/usr/sbin/taskpolicy", "-c", "utility") + if system == "Linux": + return ("nice", "-n", "10") + return () diff --git a/openhands-tools/openhands/tools/terminal/terminal/subprocess_terminal.py b/openhands-tools/openhands/tools/terminal/terminal/subprocess_terminal.py index 653d1eae7a..cfe8862bb1 100644 --- a/openhands-tools/openhands/tools/terminal/terminal/subprocess_terminal.py +++ b/openhands-tools/openhands/tools/terminal/terminal/subprocess_terminal.py @@ -35,6 +35,9 @@ from openhands.tools.terminal.metadata import CmdOutputMetadata from openhands.tools.terminal.terminal import TerminalInterface from openhands.tools.terminal.terminal.interface import parse_ctrl_key +from openhands.tools.terminal.terminal.process_priority import ( + get_process_priority_prefix, +) logger = get_logger(__name__) @@ -151,7 +154,7 @@ def initialize(self) -> None: env["PS2"] = "" env["TERM"] = "xterm-256color" - bash_cmd = [resolved_shell_path, "-i"] + bash_cmd = [*get_process_priority_prefix(), resolved_shell_path, "-i"] # Create a PTY; give the slave to the child, keep the master master_fd, slave_fd = pty.openpty() diff --git a/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py b/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py index 99452f0c64..931d1680f1 100644 --- a/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py +++ b/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py @@ -6,6 +6,7 @@ from __future__ import annotations +import shlex import threading import time import uuid @@ -28,6 +29,9 @@ build_terminal_env, normalize_terminal_env, ) +from openhands.tools.terminal.terminal.process_priority import ( + get_process_priority_prefix, +) from openhands.tools.terminal.terminal.tmux_terminal import TmuxTerminal @@ -167,13 +171,15 @@ def _create_pane(self) -> PooledTmuxTerminal: """Create a new PooledTmuxTerminal within the shared session.""" assert self._session is not None - shell_command = "/bin/bash" + shell_command = ["/bin/bash"] if self.username in ["root", "openhands"]: - shell_command = f"su {self.username} -" + shell_command = ["su", self.username, "-"] + + window_command = shlex.join((*get_process_priority_prefix(), *shell_command)) window = self._session.new_window( window_name=f"pane-{len(self._all_panes)}", - window_shell=shell_command, + window_shell=window_command, start_directory=self.work_dir, ) active_pane = window.active_pane diff --git a/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py b/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py index 059275d50d..a4f32accf1 100644 --- a/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py +++ b/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py @@ -1,5 +1,6 @@ """Tmux-based terminal backend implementation.""" +import shlex import time import uuid from collections.abc import Mapping @@ -20,6 +21,9 @@ from openhands.tools.terminal.metadata import CmdOutputMetadata from openhands.tools.terminal.terminal import TerminalInterface from openhands.tools.terminal.terminal.interface import parse_ctrl_key +from openhands.tools.terminal.terminal.process_priority import ( + get_process_priority_prefix, +) logger = get_logger(__name__) @@ -79,12 +83,12 @@ def initialize(self) -> None: env.setdefault("PAGER", "cat") # Use a dedicated socket to isolate OpenHands sessions from the user's tmux self.server = libtmux.Server(socket_name=TMUX_SOCKET_NAME, environment=env) - _shell_command = "/bin/bash" + shell_command = ["/bin/bash"] if self.username in ["root", "openhands"]: # This starts a non-login (new) shell for the given user - _shell_command = f"su {self.username} -" + shell_command = ["su", self.username, "-"] - window_command = _shell_command + window_command = shlex.join((*get_process_priority_prefix(), *shell_command)) logger.debug(f"Initializing tmux terminal with command: {window_command}") session_name = f"openhands-{self.username}-{uuid.uuid4()}" diff --git a/openhands-tools/openhands/tools/terminal/terminal/windows_terminal.py b/openhands-tools/openhands/tools/terminal/terminal/windows_terminal.py index 1751b8ca52..208317dfa3 100644 --- a/openhands-tools/openhands/tools/terminal/terminal/windows_terminal.py +++ b/openhands-tools/openhands/tools/terminal/terminal/windows_terminal.py @@ -101,6 +101,7 @@ def initialize(self) -> None: startupinfo.dwFlags |= getattr(subprocess, "STARTF_USESHOWWINDOW", 0) creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) creationflags |= getattr(subprocess, "CREATE_NO_WINDOW", 0) + creationflags |= getattr(subprocess, "BELOW_NORMAL_PRIORITY_CLASS", 0) env = build_terminal_env(self._env) env.setdefault("PYTHONIOENCODING", "utf-8") diff --git a/tests/tools/terminal/test_pool_integration.py b/tests/tools/terminal/test_pool_integration.py index 31f6621dca..913c247121 100644 --- a/tests/tools/terminal/test_pool_integration.py +++ b/tests/tools/terminal/test_pool_integration.py @@ -5,6 +5,7 @@ through the executor's __call__ interface. """ +import platform import tempfile import threading import time @@ -33,6 +34,21 @@ def pool_executor(): executor.close() +@pytest.mark.skipif( + platform.system() != "Linux", + reason="Linux niceness is only observable on Linux", +) +def test_pool_commands_inherit_lower_priority(pool_executor) -> None: + observation = pool_executor( + TerminalAction( + command="python -c 'import os; print(os.getpriority(os.PRIO_PROCESS, 0))'" + ) + ) + + assert observation.exit_code == 0 + assert int(observation.text.strip()) >= 10 + + class TestDeclaredResources: def test_pool_mode_opts_out_of_framework_locking(self, pool_executor): """In pool mode, declared_resources returns empty keys so the diff --git a/tests/tools/terminal/test_process_priority.py b/tests/tools/terminal/test_process_priority.py new file mode 100644 index 0000000000..0d976ef987 --- /dev/null +++ b/tests/tools/terminal/test_process_priority.py @@ -0,0 +1,63 @@ +"""Cross-platform process priority coverage for agent terminals.""" + +import platform +from pathlib import Path +from typing import Literal + +import pytest + +from openhands.tools.terminal.definition import TerminalAction +from openhands.tools.terminal.terminal import process_priority +from openhands.tools.terminal.terminal.factory import create_terminal_session + + +@pytest.mark.parametrize( + ("system", "expected"), + [ + pytest.param( + "Darwin", + ("/usr/sbin/taskpolicy", "-c", "utility"), + id="macos-utility-qos", + ), + pytest.param("Linux", ("nice", "-n", "10"), id="linux-niceness"), + pytest.param("Windows", (), id="windows-uses-creation-flags"), + pytest.param("FreeBSD", (), id="unsupported-platform"), + ], +) +def test_process_priority_prefix_matches_platform( + monkeypatch: pytest.MonkeyPatch, + system: str, + expected: tuple[str, ...], +) -> None: + monkeypatch.setattr(process_priority.platform, "system", lambda: system) + + assert process_priority.get_process_priority_prefix() == expected + + +@pytest.mark.skipif( + platform.system() != "Linux", + reason="Linux niceness is only observable on Linux", +) +@pytest.mark.parametrize("terminal_type", ["tmux", "subprocess"]) +def test_linux_terminal_children_inherit_lower_priority( + tmp_path: Path, + terminal_type: Literal["tmux", "subprocess"], +) -> None: + session = create_terminal_session( + work_dir=str(tmp_path), + terminal_type=terminal_type, + ) + try: + session.initialize() + observation = session.execute( + TerminalAction( + command=( + "python -c 'import os; print(os.getpriority(os.PRIO_PROCESS, 0))'" + ) + ) + ) + finally: + session.close() + + assert observation.exit_code == 0 + assert int(observation.text.strip()) >= 10 diff --git a/tests/tools/terminal/test_windows_terminal.py b/tests/tools/terminal/test_windows_terminal.py index bd24afe0eb..df60a2d584 100644 --- a/tests/tools/terminal/test_windows_terminal.py +++ b/tests/tools/terminal/test_windows_terminal.py @@ -7,6 +7,7 @@ from collections.abc import Generator from typing import cast +import psutil import pytest from pydantic import SecretStr @@ -19,6 +20,9 @@ from openhands.tools.terminal import TerminalAction, TerminalTool from openhands.tools.terminal.impl import TerminalExecutor from openhands.tools.terminal.terminal import TerminalSession, create_terminal_session +from openhands.tools.terminal.terminal.windows_terminal import ( + WindowsTerminal, +) pytestmark = pytest.mark.skipif( @@ -93,6 +97,17 @@ def test_basic_command_execution(windows_session) -> None: assert "Hello from Windows terminal" in obs.text +def test_terminal_process_runs_below_normal_priority(windows_session) -> None: + terminal = cast(WindowsTerminal, windows_session.terminal) + assert terminal.process is not None + process = terminal.process + try: + assert psutil.Process(process.pid).nice() == psutil.BELOW_NORMAL_PRIORITY_CLASS + finally: + process.terminate() + process.wait(timeout=5) + + @pytest.mark.parametrize( ("command", "expected"), [ From 024941d8cdb4c2fa701ee4c1cee179b787445feb Mon Sep 17 00:00:00 2001 From: primorLee <252785625+primorLee@users.noreply.github.com> Date: Sat, 22 Aug 2026 04:31:36 +0800 Subject: [PATCH 2/2] fix(terminal): make priority policy resilient Co-authored-by: openhands --- .../openhands/tools/terminal/README.md | 5 +++ .../terminal/terminal/process_priority.py | 24 +++++++++++- .../terminal/terminal/subprocess_terminal.py | 2 +- .../tools/terminal/terminal/tmux_pane_pool.py | 7 +++- .../tools/terminal/terminal/tmux_terminal.py | 2 +- .../terminal/terminal/windows_terminal.py | 9 ++++- tests/tools/terminal/test_pool_integration.py | 14 ++++++- tests/tools/terminal/test_process_priority.py | 37 +++++++++++++++++-- tests/tools/terminal/test_windows_terminal.py | 16 ++++++-- 9 files changed, 100 insertions(+), 16 deletions(-) diff --git a/openhands-tools/openhands/tools/terminal/README.md b/openhands-tools/openhands/tools/terminal/README.md index fe84b94bc0..a4ab5ea805 100644 --- a/openhands-tools/openhands/tools/terminal/README.md +++ b/openhands-tools/openhands/tools/terminal/README.md @@ -63,6 +63,11 @@ Raw secret values passed through `env` become session-scoped environment variables. Use the SDK secret registry when you need command-scoped secret injection behavior. +Agent terminal processes run at lower CPU priority by default so CPU-heavy +commands do not starve the agent server. Set +`OH_TERMINAL_PROCESS_PRIORITY=none` in the terminal `env` to retain the parent +process priority for benchmarks or priority-sensitive development servers. + ## Usage Examples ### Basic Usage diff --git a/openhands-tools/openhands/tools/terminal/terminal/process_priority.py b/openhands-tools/openhands/tools/terminal/terminal/process_priority.py index 0abfcae649..0d643b1090 100644 --- a/openhands-tools/openhands/tools/terminal/terminal/process_priority.py +++ b/openhands-tools/openhands/tools/terminal/terminal/process_priority.py @@ -1,13 +1,33 @@ """Platform-specific process priority policy for agent terminals.""" +import os import platform +import shutil +from collections.abc import Mapping -def get_process_priority_prefix() -> tuple[str, ...]: +TERMINAL_PROCESS_PRIORITY_ENV = "OH_TERMINAL_PROCESS_PRIORITY" + + +def should_lower_process_priority(env: Mapping[str, str] | None = None) -> bool: + """Return whether agent terminal processes should run at lower priority.""" + source = os.environ if env is None else env + value = source.get(TERMINAL_PROCESS_PRIORITY_ENV) + return value is None or value.strip().lower() != "none" + + +def get_process_priority_prefix( + env: Mapping[str, str] | None = None, +) -> tuple[str, ...]: """Return an argv prefix that lowers an agent terminal's CPU priority.""" + if not should_lower_process_priority(env): + return () + system = platform.system() if system == "Darwin": return ("/usr/sbin/taskpolicy", "-c", "utility") if system == "Linux": - return ("nice", "-n", "10") + nice_path = shutil.which("nice") + if nice_path is not None: + return (os.path.abspath(nice_path), "-n", "10") return () diff --git a/openhands-tools/openhands/tools/terminal/terminal/subprocess_terminal.py b/openhands-tools/openhands/tools/terminal/terminal/subprocess_terminal.py index cfe8862bb1..0971ec000f 100644 --- a/openhands-tools/openhands/tools/terminal/terminal/subprocess_terminal.py +++ b/openhands-tools/openhands/tools/terminal/terminal/subprocess_terminal.py @@ -154,7 +154,7 @@ def initialize(self) -> None: env["PS2"] = "" env["TERM"] = "xterm-256color" - bash_cmd = [*get_process_priority_prefix(), resolved_shell_path, "-i"] + bash_cmd = [*get_process_priority_prefix(env), resolved_shell_path, "-i"] # Create a PTY; give the slave to the child, keep the master master_fd, slave_fd = pty.openpty() diff --git a/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py b/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py index 931d1680f1..10cbb9f7fb 100644 --- a/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py +++ b/openhands-tools/openhands/tools/terminal/terminal/tmux_pane_pool.py @@ -175,7 +175,12 @@ def _create_pane(self) -> PooledTmuxTerminal: if self.username in ["root", "openhands"]: shell_command = ["su", self.username, "-"] - window_command = shlex.join((*get_process_priority_prefix(), *shell_command)) + window_command = shlex.join( + ( + *get_process_priority_prefix(build_terminal_env(self.env)), + *shell_command, + ) + ) window = self._session.new_window( window_name=f"pane-{len(self._all_panes)}", diff --git a/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py b/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py index a4f32accf1..c19c9bd64c 100644 --- a/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py +++ b/openhands-tools/openhands/tools/terminal/terminal/tmux_terminal.py @@ -88,7 +88,7 @@ def initialize(self) -> None: # This starts a non-login (new) shell for the given user shell_command = ["su", self.username, "-"] - window_command = shlex.join((*get_process_priority_prefix(), *shell_command)) + window_command = shlex.join((*get_process_priority_prefix(env), *shell_command)) logger.debug(f"Initializing tmux terminal with command: {window_command}") session_name = f"openhands-{self.username}-{uuid.uuid4()}" diff --git a/openhands-tools/openhands/tools/terminal/terminal/windows_terminal.py b/openhands-tools/openhands/tools/terminal/terminal/windows_terminal.py index 208317dfa3..b5baec5ef0 100644 --- a/openhands-tools/openhands/tools/terminal/terminal/windows_terminal.py +++ b/openhands-tools/openhands/tools/terminal/terminal/windows_terminal.py @@ -26,6 +26,9 @@ TerminalInterface, parse_ctrl_key, ) +from openhands.tools.terminal.terminal.process_priority import ( + should_lower_process_priority, +) logger = get_logger(__name__) @@ -92,6 +95,8 @@ def initialize(self) -> None: if self._initialized: return + env = build_terminal_env(self._env) + startupinfo = None creationflags = 0 if platform.system() == "Windows": @@ -101,9 +106,9 @@ def initialize(self) -> None: startupinfo.dwFlags |= getattr(subprocess, "STARTF_USESHOWWINDOW", 0) creationflags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) creationflags |= getattr(subprocess, "CREATE_NO_WINDOW", 0) - creationflags |= getattr(subprocess, "BELOW_NORMAL_PRIORITY_CLASS", 0) + if should_lower_process_priority(env): + creationflags |= getattr(subprocess, "BELOW_NORMAL_PRIORITY_CLASS", 0) - env = build_terminal_env(self._env) env.setdefault("PYTHONIOENCODING", "utf-8") env.setdefault("PYTHONUTF8", "1") diff --git a/tests/tools/terminal/test_pool_integration.py b/tests/tools/terminal/test_pool_integration.py index 913c247121..5cba023322 100644 --- a/tests/tools/terminal/test_pool_integration.py +++ b/tests/tools/terminal/test_pool_integration.py @@ -6,10 +6,12 @@ """ import platform +import re import tempfile import threading import time +import psutil import pytest from openhands.sdk.tool import DeclaredResources @@ -39,14 +41,22 @@ def pool_executor(): reason="Linux niceness is only observable on Linux", ) def test_pool_commands_inherit_lower_priority(pool_executor) -> None: + parent_priority = int(psutil.Process().nice()) observation = pool_executor( TerminalAction( - command="python -c 'import os; print(os.getpriority(os.PRIO_PROCESS, 0))'" + command=( + "python -c 'import os; " + 'print("OH_PRIORITY=" + ' + "str(os.getpriority(os.PRIO_PROCESS, 0)))'" + ) ) ) assert observation.exit_code == 0 - assert int(observation.text.strip()) >= 10 + match = re.search(r"OH_PRIORITY=(-?\d+)", observation.text) + assert match is not None, observation.text + expected_priority = min(19, parent_priority + 10) + assert int(match.group(1)) >= expected_priority class TestDeclaredResources: diff --git a/tests/tools/terminal/test_process_priority.py b/tests/tools/terminal/test_process_priority.py index 0d976ef987..7d62ec93e1 100644 --- a/tests/tools/terminal/test_process_priority.py +++ b/tests/tools/terminal/test_process_priority.py @@ -1,9 +1,11 @@ """Cross-platform process priority coverage for agent terminals.""" import platform +import re from pathlib import Path from typing import Literal +import psutil import pytest from openhands.tools.terminal.definition import TerminalAction @@ -19,7 +21,7 @@ ("/usr/sbin/taskpolicy", "-c", "utility"), id="macos-utility-qos", ), - pytest.param("Linux", ("nice", "-n", "10"), id="linux-niceness"), + pytest.param("Linux", ("/usr/bin/nice", "-n", "10"), id="linux-niceness"), pytest.param("Windows", (), id="windows-uses-creation-flags"), pytest.param("FreeBSD", (), id="unsupported-platform"), ], @@ -30,10 +32,33 @@ def test_process_priority_prefix_matches_platform( expected: tuple[str, ...], ) -> None: monkeypatch.setattr(process_priority.platform, "system", lambda: system) + monkeypatch.setattr( + process_priority.shutil, "which", lambda command: f"/usr/bin/{command}" + ) + monkeypatch.setattr(process_priority.os.path, "abspath", lambda path: path) assert process_priority.get_process_priority_prefix() == expected +def test_linux_priority_falls_back_when_nice_is_unavailable(monkeypatch) -> None: + monkeypatch.setattr(process_priority.platform, "system", lambda: "Linux") + monkeypatch.setattr(process_priority.shutil, "which", lambda _command: None) + + assert process_priority.get_process_priority_prefix() == () + + +@pytest.mark.parametrize("system", ["Darwin", "Linux"]) +def test_none_setting_disables_process_priority_policy( + monkeypatch: pytest.MonkeyPatch, + system: str, +) -> None: + monkeypatch.setattr(process_priority.platform, "system", lambda: system) + monkeypatch.setattr(process_priority.shutil, "which", lambda _: "/usr/bin/nice") + env = {process_priority.TERMINAL_PROCESS_PRIORITY_ENV: "none"} + + assert process_priority.get_process_priority_prefix(env) == () + + @pytest.mark.skipif( platform.system() != "Linux", reason="Linux niceness is only observable on Linux", @@ -47,12 +72,15 @@ def test_linux_terminal_children_inherit_lower_priority( work_dir=str(tmp_path), terminal_type=terminal_type, ) + parent_priority = int(psutil.Process().nice()) try: session.initialize() observation = session.execute( TerminalAction( command=( - "python -c 'import os; print(os.getpriority(os.PRIO_PROCESS, 0))'" + "python -c 'import os; " + 'print("OH_PRIORITY=" + ' + "str(os.getpriority(os.PRIO_PROCESS, 0)))'" ) ) ) @@ -60,4 +88,7 @@ def test_linux_terminal_children_inherit_lower_priority( session.close() assert observation.exit_code == 0 - assert int(observation.text.strip()) >= 10 + match = re.search(r"OH_PRIORITY=(-?\d+)", observation.text) + assert match is not None, observation.text + expected_priority = min(19, parent_priority + 10) + assert int(match.group(1)) >= expected_priority diff --git a/tests/tools/terminal/test_windows_terminal.py b/tests/tools/terminal/test_windows_terminal.py index df60a2d584..b0cd91eb83 100644 --- a/tests/tools/terminal/test_windows_terminal.py +++ b/tests/tools/terminal/test_windows_terminal.py @@ -100,12 +100,20 @@ def test_basic_command_execution(windows_session) -> None: def test_terminal_process_runs_below_normal_priority(windows_session) -> None: terminal = cast(WindowsTerminal, windows_session.terminal) assert terminal.process is not None - process = terminal.process try: - assert psutil.Process(process.pid).nice() == psutil.BELOW_NORMAL_PRIORITY_CLASS + observation = windows_session.execute( + TerminalAction( + command=( + 'python -c "import psutil; ' + "print('OH_PRIORITY=' + str(psutil.Process().nice()))\"" + ) + ) + ) + assert observation.exit_code == 0 + assert f"OH_PRIORITY={psutil.BELOW_NORMAL_PRIORITY_CLASS}" in observation.text finally: - process.terminate() - process.wait(timeout=5) + terminal.process.terminate() + terminal.process.wait(timeout=5) @pytest.mark.parametrize(