From 1416d8495f79b05ff93c6fca0f33637719c0e597 Mon Sep 17 00:00:00 2001 From: MagMueller Date: Sat, 15 Aug 2026 08:17:28 -0700 Subject: [PATCH 1/2] feat: add Browser Use CLI skill Co-authored-by: openhands --- README.md | 5 +- marketplaces/openhands-extensions.json | 14 ++ skills/browser-use/.claude-plugin | 1 + skills/browser-use/.codex-plugin | 1 + skills/browser-use/.plugin/plugin.json | 20 +++ skills/browser-use/README.md | 26 +++ skills/browser-use/SKILL.md | 210 +++++++++++++++++++++++++ skills/index.js | 14 ++ 8 files changed, 289 insertions(+), 2 deletions(-) create mode 120000 skills/browser-use/.claude-plugin create mode 120000 skills/browser-use/.codex-plugin create mode 100644 skills/browser-use/.plugin/plugin.json create mode 100644 skills/browser-use/README.md create mode 100644 skills/browser-use/SKILL.md diff --git a/README.md b/README.md index 424440b1..fc7be96a 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ The JS and Python versions are kept in lock-step by `release-please` and guarded ## Extensions Catalog -This repository contains **2 marketplace(s)** with **64 extensions** (54 skills, 10 plugins). +This repository contains **2 marketplace(s)** with **65 extensions** (55 skills, 10 plugins). ### large-codebase @@ -108,7 +108,7 @@ OpenHands skills for interacting, improving, and refactoring large codebases Official skills and plugins for OpenHands — the open-source AI software engineer. -**60 extensions** (52 skills, 8 plugins) +**61 extensions** (53 skills, 8 plugins) | Name | Type | Description | Commands | |------|------|-------------|----------| @@ -121,6 +121,7 @@ Official skills and plugins for OpenHands — the open-source AI software engine | bitbucket | skill | Bitbucket integration hub. Detects whether the repository is on Bitbucket Cloud or Bitbucket Data Center and directs ... | — | | bitbucket-cloud | skill | Bitbucket Cloud (bitbucket.org) specifics — authenticate with BITBUCKET_TOKEN, use the REST API v2, workspace/repo_sl... | — | | bitbucket-data-center | skill | Bitbucket Data Center (self-hosted Bitbucket Server) specifics — authenticate with BITBUCKET_DATA_CENTER_TOKEN, use t... | — | +| browser-use | skill | Direct browser control with Browser Use CLI 3.0 for web interaction, automation, scraping, testing, screenshots, and ... | — | | city-weather | plugin | Get current weather, time, and precipitation forecast for any city using the free Open-Meteo API. Provides slash comm... | — | | code-review | skill | Rigorous code review focusing on data structures, simplicity, security, pragmatism, and risk/safety evaluation. Provi... | `/codereview`, `/codereview-roasted` | | code-simplifier | skill | Simplifies and refines code across three dimensions - code reuse, code quality, and efficiency - while preserving all... | `/simplify` | diff --git a/marketplaces/openhands-extensions.json b/marketplaces/openhands-extensions.json index dfed7887..fed72c3e 100644 --- a/marketplaces/openhands-extensions.json +++ b/marketplaces/openhands-extensions.json @@ -138,6 +138,20 @@ "pull-request" ] }, + { + "name": "browser-use", + "source": "./skills/browser-use", + "description": "Direct browser control with Browser Use CLI 3.0 for web interaction, automation, scraping, testing, screenshots, and site or app work.", + "category": "environment", + "keywords": [ + "browser-use", + "browser", + "cdp", + "automation", + "scraping", + "testing" + ] + }, { "name": "bitbucket-cloud", "source": "./skills/bitbucket-cloud", diff --git a/skills/browser-use/.claude-plugin b/skills/browser-use/.claude-plugin new file mode 120000 index 00000000..665797f0 --- /dev/null +++ b/skills/browser-use/.claude-plugin @@ -0,0 +1 @@ +.plugin \ No newline at end of file diff --git a/skills/browser-use/.codex-plugin b/skills/browser-use/.codex-plugin new file mode 120000 index 00000000..665797f0 --- /dev/null +++ b/skills/browser-use/.codex-plugin @@ -0,0 +1 @@ +.plugin \ No newline at end of file diff --git a/skills/browser-use/.plugin/plugin.json b/skills/browser-use/.plugin/plugin.json new file mode 100644 index 00000000..d0191567 --- /dev/null +++ b/skills/browser-use/.plugin/plugin.json @@ -0,0 +1,20 @@ +{ + "name": "browser-use", + "version": "1.0.0", + "description": "Direct browser control with Browser Use CLI 3.0 for web interaction, automation, scraping, testing, screenshots, and site or app work.", + "author": { + "name": "Browser Use", + "email": "support@browser-use.com" + }, + "homepage": "https://browser-use.com", + "repository": "https://github.com/browser-use/browser-use", + "license": "MIT", + "keywords": [ + "browser-use", + "browser", + "cdp", + "automation", + "scraping", + "testing" + ] +} diff --git a/skills/browser-use/README.md b/skills/browser-use/README.md new file mode 100644 index 00000000..222b1888 --- /dev/null +++ b/skills/browser-use/README.md @@ -0,0 +1,26 @@ +# Browser Use CLI 3.0 + +Give OpenHands direct CDP browser control through the Browser Use CLI instead +of adding a large browser tool schema to every model turn. + +The skill bootstraps the current `browser-use` package with `uv`, supports local +Chrome and Browser Use Cloud, and gives the agent operational guidance for page +interaction, authentication boundaries, remote-session cleanup, and recordings. + +## Try it + +Install this skill from the OpenHands extensions marketplace, then ask: + +> Use Browser Use to open the Browser Use GitHub repository and report its +> latest release. + +For a headless OpenHands environment, authenticate first with: + +```bash +browser-use auth login --device-code +``` + +Project: https://github.com/browser-use/browser-use + +CLI and interaction guides: +https://github.com/browser-use/browser-harness diff --git a/skills/browser-use/SKILL.md b/skills/browser-use/SKILL.md new file mode 100644 index 00000000..b51cdffe --- /dev/null +++ b/skills/browser-use/SKILL.md @@ -0,0 +1,210 @@ +--- +name: browser-use +description: Direct browser control with Browser Use CLI 3.0 for web interaction, automation, scraping, testing, screenshots, and site or app work. Use when a task needs clicks, typing, navigation, a logged-in browser, JavaScript rendering, or a bot-protected page. +triggers: +- browser-use +- browser automation +- web automation +- website interaction +- UI testing +- scraping +--- + +# Browser Use CLI 3.0 + +Use Browser Use through OpenHands' terminal instead of the built-in structured +browser tools while this skill is active. Browser Use keeps the browser API out +of the model's tool schema and exposes direct CDP helpers through a Python CLI. + +## Setup + +Check that the current Browser Use CLI is available: + +```bash +browser-use --help 2>/dev/null | grep -q "Read SKILL.md" || \ + uv tool install --python 3.12 --upgrade --force browser-use +``` + +For setup, install, or connection problems, run `browser-use --doctor` and read +https://github.com/browser-use/browser-harness/blob/main/install.md. + +## When Not to Use + +A basic fetch of public information needs no browser. If a plain HTTP request +can read a public page, API, or documentation, use `curl` or another fetch tool. +Use Browser Use when the task needs interaction, the user's logged-in session, +JavaScript rendering, or a bot-protected page. If a direct fetch fails or +returns a shell page, escalate to the browser. + +Domain skills are off by default. Set `BH_DOMAIN_SKILLS=1` to enable them; see +the bottom section. + +If `BH_DOMAIN_SKILLS=1` and the task is site-specific, read every file in the +matching `$BH_AGENT_WORKSPACE/domain-skills//` directory before inventing +an approach. + +## Usage + +```bash +browser-use <<'PY' +print(page_info()) +PY +``` + +- Invoke as `browser-use`. Use heredocs for multi-line commands. +- Helpers are pre-imported. The CLI starts its daemon before execution. +- First navigation is `new_tab(url)`, not `goto_url(url)`. +- The normal local flow attaches to a running Chrome or Chromium CDP endpoint. + +## Local Chrome + +If the daemon cannot connect, run diagnostics: + +```bash +browser-use --doctor +``` + +If Chrome is not running, the harness launches it automatically and retries. + +If Chrome is running but remote debugging is not enabled, the harness opens: + +```text +chrome://inspect/#remote-debugging +``` + +On macOS, when Chrome asks for remote-debugging permission, run: + +```text +browser-use mac-approve +``` + +Continue when it returns `ready`; otherwise follow its printed instruction. + +## Remote Browsers + +Use Browser Use Cloud for headless servers, parallel sub-agents, or isolated +work. Each cloud browser is a fresh, isolated managed Chrome instance. Prefer +one when: + +- The user wants multiple concurrent tasks. One cloud browser per task avoids + tabs and focus fighting in a shared local browser. +- Captchas or blocking are likely. Cloud browsers use clean managed IPs and + stealth settings, keeping the user's own IP and browser out of the task. + +Authenticate once: + +```bash +browser-use auth login --device-code +``` + +Or import a key safely: + +```bash +printf '%s' "$BROWSER_USE_API_KEY" | browser-use auth login --api-key-stdin +``` + +Pick a short made-up name; `r7k2` below is only a placeholder: + +```bash +browser-use <<'PY' +start_remote_daemon("r7k2") +PY + +BU_NAME=r7k2 browser-use <<'PY' +new_tab("https://example.com") +print(page_info()) +PY +``` + +When the task is done and a cloud browser is still running, ask directly: +"Should I close this browser now?" If yes, run `stop_remote_daemon(name)`. +Remote daemons bill until they stop or time out. Always use the same `BU_NAME` +after starting a named remote daemon. + +Cloud profile cookie sync reference: +https://github.com/browser-use/browser-harness/blob/main/interaction-skills/profile-sync.md. + +## Page Workflow + +- Prefer the accessibility tree over screenshots for finding elements: + `cdp("Accessibility.getFullAXTree")["nodes"]` includes roles, names, and + `backendDOMNodeId` values. Filter in Python before printing because it can + contain thousands of nodes. +- Resolve an accessibility node's box with + `cdp("DOM.getBoxModel", backendNodeId=n)`, click its center with + `click_at_xy(x, y)`, and verify the result with targeted `js(...)` or + `page_info()` output. +- Fall back to raw HTML through `js(...)` only when the accessibility tree lacks + the element, such as canvas or unusual widgets. Use screenshots when layout + or imagery matters. +- After navigation, call `wait_for_load()`. +- If the current tab is stale or internal, call `ensure_real_tab()`. +- Use `js(...)` for DOM inspection or extraction when coordinates are the wrong + tool. +- Stop at login walls. You may use available SSO when Chrome is already signed + in, but stop for passwords, MFA, consent, or an ambiguous account choice. +- Raw CDP is available with `cdp("Domain.method", ...)`. + +## Recordings and Videos + +Fresh installs do not record. Users can enable local background traces: + +```bash +browser-use recordings enable +browser-use recordings disable +browser-use recordings +``` + +`BH_RECORD=1` or `BH_RECORD=0` overrides the preference for one process. A +request to record, show, demo, or make a video opts in that task; significant +work alone does not. + +Before browser work, call `start_recording(name, title=...)`, retain its exact +returned directory, and call `stop_recording()` after verification. Never +replace that path with `recordings --latest`. For a request made after the task, +use: + +```bash +browser-use recordings --latest +``` + +Use it only if timestamps and pages match. Otherwise say the work was not +captured. Never reenact completed work. For a video, follow +https://github.com/browser-use/browser-harness/blob/main/interaction-skills/make-video.md. + +## Interaction Skills + +If a browser mechanic is blocking progress, check the focused guides at +https://github.com/browser-use/browser-harness/tree/main/interaction-skills. +They cover cookies, cross-origin iframes, dialogs, downloads, drag and drop, +dropdowns, uploads, scrolling, screenshots, tabs, and viewports. + +## Design Constraints + +- Coordinate clicks are the default. CDP mouse events pass through iframe, + shadow DOM, and cross-origin boundaries at the compositor level. +- Keep the connection model simple: use the default daemon, `BU_NAME`, + `BU_CDP_URL`, `BU_CDP_WS`, or `start_remote_daemon(...)`. +- Keep core helpers short. Put task-specific helper additions in + `$BH_AGENT_WORKSPACE/agent_helpers.py`. + +## Gotchas + +- `chrome://inspect/#remote-debugging` must be enabled for local Chrome control. +- On macOS, if Chrome shows an "Allow remote debugging?" popup, run + `browser-use mac-approve`. Do not poll in a loop because the daemon holds one + connection. +- Omnibox popups are not real work tabs. +- CDP target order is not Chrome's visible tab-strip order. +- `BU_CDP_URL` is an HTTP DevTools endpoint; the daemon resolves it to a + WebSocket. +- Ask before leaving cloud browsers running. Stop them with + `stop_remote_daemon(name)` or `PATCH /browsers/{id} {"action":"stop"}`. + +## Domain Skills + +This section applies only when `BH_DOMAIN_SKILLS=1`. + +When enabled, search `$BH_AGENT_WORKSPACE/domain-skills//` before +inventing an approach. `goto_url(...)` returns up to 10 skill filenames for the +navigated host. diff --git a/skills/index.js b/skills/index.js index bc9d49a2..5868c2e4 100644 --- a/skills/index.js +++ b/skills/index.js @@ -97,6 +97,20 @@ export const SKILLS_CATALOG = [ "content": "You are working with **Bitbucket Data Center** (self-hosted Bitbucket Server). You have\naccess to an environment variable, `BITBUCKET_DATA_CENTER_TOKEN`, which contains a basic\nauth token in the format `username:your-token` that allows you to interact with the git\nrepository and the REST API.\n\n> Environment variable names are case-sensitive. If `BITBUCKET_DATA_CENTER_TOKEN` is not\n> present, use whichever case variant actually exists (for example\n> `bitbucket_data_center_token`). Run `env | grep -i 'bitbucket_data_center'` to find it.\n\n- REST API base URL: `https://{domain}/rest/api/1.0`\n- Repository identifier format: `PROJECT/repo_slug` (project key, slash, repo slug)\n\nYou can use this token to interact with the Bitbucket Data Center REST API:\n```bash\ncurl -u \"${BITBUCKET_DATA_CENTER_TOKEN}\" https://{domain}/rest/api/1.0/...\n```\n\n\nALWAYS use the Bitbucket Data Center API for operations instead of a web browser.\nALWAYS use the `create_bitbucket_data_center_pr` tool to open a pull request\n\n\nIf you encounter authentication issues when pushing to Bitbucket Data Center (such as password prompts or permission errors), the old token may have expired. In such case, update the remote URL to include the current token: `git remote set-url origin https://${BITBUCKET_DATA_CENTER_TOKEN}@{domain}/scm/{project_lower}/{repo}.git`\n\nThe token is a `username:token` pair, so if the username or token contains characters that are reserved in URLs (such as `@`), split on the first `:` and URL-encode each part before embedding it in a remote:\n\n```bash\nBB_USER=\"${BITBUCKET_DATA_CENTER_TOKEN%%:*}\" && \\\nBB_PASS=\"${BITBUCKET_DATA_CENTER_TOKEN#*:}\" && \\\nENCODED_USER=$(python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=\"\"))' \"$BB_USER\") && \\\nENCODED_PASS=$(python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=\"\"))' \"$BB_PASS\") && \\\ngit remote set-url origin \"https://${ENCODED_USER}:${ENCODED_PASS}@{domain}/scm/{project_lower}/{repo}.git\"\n```\n\nHere are some instructions for pushing, but ONLY do this if the user asks you to:\n* NEVER push directly to the `main` or `master` branch\n* Git config (username and email) is pre-set. Do not modify.\n* You may already be on a branch starting with `openhands-workspace`. Create a new branch with a better name before pushing.\n* Use the `create_bitbucket_data_center_pr` tool to create a pull request, if you haven't already\n* Once you've created your own branch or a pull request, continue to update it. Do NOT create a new one unless you are explicitly asked to. Update the PR title and description as necessary, but don't change the branch name.\n* Use the main branch as the base branch, unless the user requests otherwise\n* After opening or updating a pull request, send the user a short message with a link to the pull request.\n* Do NOT mark a pull request as ready to review unless the user explicitly says so\n* Do all of the above in as few steps as possible. E.g. you could push changes with one step by running the following bash commands:\n```bash\ngit remote -v && git branch # to find the current org, repo and branch\ngit checkout -b create-widget && git add . && git commit -m \"Create widget\" && git push -u origin create-widget\n```", "category": "code-hosting" }, + { + "name": "browser-use", + "description": "Direct browser control with Browser Use CLI 3.0 for web interaction, automation, scraping, testing, screenshots, and site or app work. Use when a task needs clicks, typing, navigation, a logged-in browser, JavaScript rendering, or a bot-protected page.", + "triggers": [ + "browser-use", + "browser automation", + "web automation", + "website interaction", + "UI testing", + "scraping" + ], + "content": "# Browser Use CLI 3.0\n\nUse Browser Use through OpenHands' terminal instead of the built-in structured\nbrowser tools while this skill is active. Browser Use keeps the browser API out\nof the model's tool schema and exposes direct CDP helpers through a Python CLI.\n\n## Setup\n\nCheck that the current Browser Use CLI is available:\n\n```bash\nbrowser-use --help 2>/dev/null | grep -q \"Read SKILL.md\" || \\\n uv tool install --python 3.12 --upgrade --force browser-use\n```\n\nFor setup, install, or connection problems, run `browser-use --doctor` and read\nhttps://github.com/browser-use/browser-harness/blob/main/install.md.\n\n## When Not to Use\n\nA basic fetch of public information needs no browser. If a plain HTTP request\ncan read a public page, API, or documentation, use `curl` or another fetch tool.\nUse Browser Use when the task needs interaction, the user's logged-in session,\nJavaScript rendering, or a bot-protected page. If a direct fetch fails or\nreturns a shell page, escalate to the browser.\n\nDomain skills are off by default. Set `BH_DOMAIN_SKILLS=1` to enable them; see\nthe bottom section.\n\nIf `BH_DOMAIN_SKILLS=1` and the task is site-specific, read every file in the\nmatching `$BH_AGENT_WORKSPACE/domain-skills//` directory before inventing\nan approach.\n\n## Usage\n\n```bash\nbrowser-use <<'PY'\nprint(page_info())\nPY\n```\n\n- Invoke as `browser-use`. Use heredocs for multi-line commands.\n- Helpers are pre-imported. The CLI starts its daemon before execution.\n- First navigation is `new_tab(url)`, not `goto_url(url)`.\n- The normal local flow attaches to a running Chrome or Chromium CDP endpoint.\n\n## Local Chrome\n\nIf the daemon cannot connect, run diagnostics:\n\n```bash\nbrowser-use --doctor\n```\n\nIf Chrome is not running, the harness launches it automatically and retries.\n\nIf Chrome is running but remote debugging is not enabled, the harness opens:\n\n```text\nchrome://inspect/#remote-debugging\n```\n\nOn macOS, when Chrome asks for remote-debugging permission, run:\n\n```text\nbrowser-use mac-approve\n```\n\nContinue when it returns `ready`; otherwise follow its printed instruction.\n\n## Remote Browsers\n\nUse Browser Use Cloud for headless servers, parallel sub-agents, or isolated\nwork. Each cloud browser is a fresh, isolated managed Chrome instance. Prefer\none when:\n\n- The user wants multiple concurrent tasks. One cloud browser per task avoids\n tabs and focus fighting in a shared local browser.\n- Captchas or blocking are likely. Cloud browsers use clean managed IPs and\n stealth settings, keeping the user's own IP and browser out of the task.\n\nAuthenticate once:\n\n```bash\nbrowser-use auth login --device-code\n```\n\nOr import a key safely:\n\n```bash\nprintf '%s' \"$BROWSER_USE_API_KEY\" | browser-use auth login --api-key-stdin\n```\n\nPick a short made-up name; `r7k2` below is only a placeholder:\n\n```bash\nbrowser-use <<'PY'\nstart_remote_daemon(\"r7k2\")\nPY\n\nBU_NAME=r7k2 browser-use <<'PY'\nnew_tab(\"https://example.com\")\nprint(page_info())\nPY\n```\n\nWhen the task is done and a cloud browser is still running, ask directly:\n\"Should I close this browser now?\" If yes, run `stop_remote_daemon(name)`.\nRemote daemons bill until they stop or time out. Always use the same `BU_NAME`\nafter starting a named remote daemon.\n\nCloud profile cookie sync reference:\nhttps://github.com/browser-use/browser-harness/blob/main/interaction-skills/profile-sync.md.\n\n## Page Workflow\n\n- Prefer the accessibility tree over screenshots for finding elements:\n `cdp(\"Accessibility.getFullAXTree\")[\"nodes\"]` includes roles, names, and\n `backendDOMNodeId` values. Filter in Python before printing because it can\n contain thousands of nodes.\n- Resolve an accessibility node's box with\n `cdp(\"DOM.getBoxModel\", backendNodeId=n)`, click its center with\n `click_at_xy(x, y)`, and verify the result with targeted `js(...)` or\n `page_info()` output.\n- Fall back to raw HTML through `js(...)` only when the accessibility tree lacks\n the element, such as canvas or unusual widgets. Use screenshots when layout\n or imagery matters.\n- After navigation, call `wait_for_load()`.\n- If the current tab is stale or internal, call `ensure_real_tab()`.\n- Use `js(...)` for DOM inspection or extraction when coordinates are the wrong\n tool.\n- Stop at login walls. You may use available SSO when Chrome is already signed\n in, but stop for passwords, MFA, consent, or an ambiguous account choice.\n- Raw CDP is available with `cdp(\"Domain.method\", ...)`.\n\n## Recordings and Videos\n\nFresh installs do not record. Users can enable local background traces:\n\n```bash\nbrowser-use recordings enable\nbrowser-use recordings disable\nbrowser-use recordings\n```\n\n`BH_RECORD=1` or `BH_RECORD=0` overrides the preference for one process. A\nrequest to record, show, demo, or make a video opts in that task; significant\nwork alone does not.\n\nBefore browser work, call `start_recording(name, title=...)`, retain its exact\nreturned directory, and call `stop_recording()` after verification. Never\nreplace that path with `recordings --latest`. For a request made after the task,\nuse:\n\n```bash\nbrowser-use recordings --latest\n```\n\nUse it only if timestamps and pages match. Otherwise say the work was not\ncaptured. Never reenact completed work. For a video, follow\nhttps://github.com/browser-use/browser-harness/blob/main/interaction-skills/make-video.md.\n\n## Interaction Skills\n\nIf a browser mechanic is blocking progress, check the focused guides at\nhttps://github.com/browser-use/browser-harness/tree/main/interaction-skills.\nThey cover cookies, cross-origin iframes, dialogs, downloads, drag and drop,\ndropdowns, uploads, scrolling, screenshots, tabs, and viewports.\n\n## Design Constraints\n\n- Coordinate clicks are the default. CDP mouse events pass through iframe,\n shadow DOM, and cross-origin boundaries at the compositor level.\n- Keep the connection model simple: use the default daemon, `BU_NAME`,\n `BU_CDP_URL`, `BU_CDP_WS`, or `start_remote_daemon(...)`.\n- Keep core helpers short. Put task-specific helper additions in\n `$BH_AGENT_WORKSPACE/agent_helpers.py`.\n\n## Gotchas\n\n- `chrome://inspect/#remote-debugging` must be enabled for local Chrome control.\n- On macOS, if Chrome shows an \"Allow remote debugging?\" popup, run\n `browser-use mac-approve`. Do not poll in a loop because the daemon holds one\n connection.\n- Omnibox popups are not real work tabs.\n- CDP target order is not Chrome's visible tab-strip order.\n- `BU_CDP_URL` is an HTTP DevTools endpoint; the daemon resolves it to a\n WebSocket.\n- Ask before leaving cloud browsers running. Stop them with\n `stop_remote_daemon(name)` or `PATCH /browsers/{id} {\"action\":\"stop\"}`.\n\n## Domain Skills\n\nThis section applies only when `BH_DOMAIN_SKILLS=1`.\n\nWhen enabled, search `$BH_AGENT_WORKSPACE/domain-skills//` before\ninventing an approach. `goto_url(...)` returns up to 10 skill filenames for the\nnavigated host.", + "category": "environment" + }, { "name": "code-review", "description": "Rigorous code review focusing on data structures, simplicity, security, pragmatism, and risk/safety evaluation. Provides brutally honest, actionable feedback on pull requests or merge requests, including a risk assessment for every review. Use when reviewing code changes.", From c22c926062d7116c1ff36e03ea261a026b75f6d5 Mon Sep 17 00:00:00 2001 From: MagMueller Date: Sat, 15 Aug 2026 08:20:44 -0700 Subject: [PATCH 2/2] fix: isolate Browser Use CLI execution Co-authored-by: openhands --- skills/browser-use/README.md | 11 +++++---- skills/browser-use/SKILL.md | 46 +++++++++++++++++++++++------------- skills/index.js | 2 +- 3 files changed, 38 insertions(+), 21 deletions(-) diff --git a/skills/browser-use/README.md b/skills/browser-use/README.md index 222b1888..7410d7ec 100644 --- a/skills/browser-use/README.md +++ b/skills/browser-use/README.md @@ -3,9 +3,11 @@ Give OpenHands direct CDP browser control through the Browser Use CLI instead of adding a large browser tool schema to every model turn. -The skill bootstraps the current `browser-use` package with `uv`, supports local -Chrome and Browser Use Cloud, and gives the agent operational guidance for page -interaction, authentication boundaries, remote-session cleanup, and recordings. +The skill bootstraps the current `browser-use` package with `uv`, resolves the +isolated binary ahead of project virtualenv copies, and cleans inherited Python +environment variables. It supports local Chrome and Browser Use Cloud and gives +the agent operational guidance for page interaction, authentication boundaries, +remote-session cleanup, and recordings. ## Try it @@ -17,7 +19,8 @@ Install this skill from the OpenHands extensions marketplace, then ask: For a headless OpenHands environment, authenticate first with: ```bash -browser-use auth login --device-code +BU_CLI="$(uv tool dir --bin)/browser-use" +env -u PYTHONPATH -u PYTHONHOME "$BU_CLI" auth login --device-code ``` Project: https://github.com/browser-use/browser-use diff --git a/skills/browser-use/SKILL.md b/skills/browser-use/SKILL.md index b51cdffe..d0ff1dfe 100644 --- a/skills/browser-use/SKILL.md +++ b/skills/browser-use/SKILL.md @@ -18,14 +18,19 @@ of the model's tool schema and exposes direct CDP helpers through a Python CLI. ## Setup -Check that the current Browser Use CLI is available: +Install or upgrade the current Browser Use CLI in an isolated `uv` tool +environment: ```bash -browser-use --help 2>/dev/null | grep -q "Read SKILL.md" || \ - uv tool install --python 3.12 --upgrade --force browser-use +uv tool install --python 3.12 --upgrade --force browser-use +BU_CLI="$(uv tool dir --bin)/browser-use" +env -u PYTHONPATH -u PYTHONHOME "$BU_CLI" --help ``` -For setup, install, or connection problems, run `browser-use --doctor` and read +Use the exact `BU_CLI` path and clean child environment for every invocation. +This prevents a project virtualenv's older `browser-use` executable or Python +packages from shadowing the isolated CLI. For setup or connection problems, run +`env -u PYTHONPATH -u PYTHONHOME "$BU_CLI" --doctor` and read https://github.com/browser-use/browser-harness/blob/main/install.md. ## When Not to Use @@ -46,12 +51,14 @@ an approach. ## Usage ```bash -browser-use <<'PY' +BU_CLI="$(uv tool dir --bin)/browser-use" +env -u PYTHONPATH -u PYTHONHOME "$BU_CLI" <<'PY' print(page_info()) PY ``` -- Invoke as `browser-use`. Use heredocs for multi-line commands. +- Invoke through `BU_CLI` with the clean environment shown above. Use heredocs + for multi-line commands. - Helpers are pre-imported. The CLI starts its daemon before execution. - First navigation is `new_tab(url)`, not `goto_url(url)`. - The normal local flow attaches to a running Chrome or Chromium CDP endpoint. @@ -61,7 +68,8 @@ PY If the daemon cannot connect, run diagnostics: ```bash -browser-use --doctor +BU_CLI="$(uv tool dir --bin)/browser-use" +env -u PYTHONPATH -u PYTHONHOME "$BU_CLI" --doctor ``` If Chrome is not running, the harness launches it automatically and retries. @@ -75,7 +83,7 @@ chrome://inspect/#remote-debugging On macOS, when Chrome asks for remote-debugging permission, run: ```text -browser-use mac-approve +env -u PYTHONPATH -u PYTHONHOME "$BU_CLI" mac-approve ``` Continue when it returns `ready`; otherwise follow its printed instruction. @@ -94,23 +102,27 @@ one when: Authenticate once: ```bash -browser-use auth login --device-code +BU_CLI="$(uv tool dir --bin)/browser-use" +env -u PYTHONPATH -u PYTHONHOME "$BU_CLI" auth login --device-code ``` Or import a key safely: ```bash -printf '%s' "$BROWSER_USE_API_KEY" | browser-use auth login --api-key-stdin +BU_CLI="$(uv tool dir --bin)/browser-use" +printf '%s' "$BROWSER_USE_API_KEY" | \ + env -u PYTHONPATH -u PYTHONHOME "$BU_CLI" auth login --api-key-stdin ``` Pick a short made-up name; `r7k2` below is only a placeholder: ```bash -browser-use <<'PY' +BU_CLI="$(uv tool dir --bin)/browser-use" +env -u PYTHONPATH -u PYTHONHOME "$BU_CLI" <<'PY' start_remote_daemon("r7k2") PY -BU_NAME=r7k2 browser-use <<'PY' +BU_NAME=r7k2 env -u PYTHONPATH -u PYTHONHOME "$BU_CLI" <<'PY' new_tab("https://example.com") print(page_info()) PY @@ -150,9 +162,10 @@ https://github.com/browser-use/browser-harness/blob/main/interaction-skills/prof Fresh installs do not record. Users can enable local background traces: ```bash -browser-use recordings enable -browser-use recordings disable -browser-use recordings +BU_CLI="$(uv tool dir --bin)/browser-use" +env -u PYTHONPATH -u PYTHONHOME "$BU_CLI" recordings enable +env -u PYTHONPATH -u PYTHONHOME "$BU_CLI" recordings disable +env -u PYTHONPATH -u PYTHONHOME "$BU_CLI" recordings ``` `BH_RECORD=1` or `BH_RECORD=0` overrides the preference for one process. A @@ -165,7 +178,8 @@ replace that path with `recordings --latest`. For a request made after the task, use: ```bash -browser-use recordings --latest +BU_CLI="$(uv tool dir --bin)/browser-use" +env -u PYTHONPATH -u PYTHONHOME "$BU_CLI" recordings --latest ``` Use it only if timestamps and pages match. Otherwise say the work was not diff --git a/skills/index.js b/skills/index.js index 5868c2e4..0ca50f59 100644 --- a/skills/index.js +++ b/skills/index.js @@ -108,7 +108,7 @@ export const SKILLS_CATALOG = [ "UI testing", "scraping" ], - "content": "# Browser Use CLI 3.0\n\nUse Browser Use through OpenHands' terminal instead of the built-in structured\nbrowser tools while this skill is active. Browser Use keeps the browser API out\nof the model's tool schema and exposes direct CDP helpers through a Python CLI.\n\n## Setup\n\nCheck that the current Browser Use CLI is available:\n\n```bash\nbrowser-use --help 2>/dev/null | grep -q \"Read SKILL.md\" || \\\n uv tool install --python 3.12 --upgrade --force browser-use\n```\n\nFor setup, install, or connection problems, run `browser-use --doctor` and read\nhttps://github.com/browser-use/browser-harness/blob/main/install.md.\n\n## When Not to Use\n\nA basic fetch of public information needs no browser. If a plain HTTP request\ncan read a public page, API, or documentation, use `curl` or another fetch tool.\nUse Browser Use when the task needs interaction, the user's logged-in session,\nJavaScript rendering, or a bot-protected page. If a direct fetch fails or\nreturns a shell page, escalate to the browser.\n\nDomain skills are off by default. Set `BH_DOMAIN_SKILLS=1` to enable them; see\nthe bottom section.\n\nIf `BH_DOMAIN_SKILLS=1` and the task is site-specific, read every file in the\nmatching `$BH_AGENT_WORKSPACE/domain-skills//` directory before inventing\nan approach.\n\n## Usage\n\n```bash\nbrowser-use <<'PY'\nprint(page_info())\nPY\n```\n\n- Invoke as `browser-use`. Use heredocs for multi-line commands.\n- Helpers are pre-imported. The CLI starts its daemon before execution.\n- First navigation is `new_tab(url)`, not `goto_url(url)`.\n- The normal local flow attaches to a running Chrome or Chromium CDP endpoint.\n\n## Local Chrome\n\nIf the daemon cannot connect, run diagnostics:\n\n```bash\nbrowser-use --doctor\n```\n\nIf Chrome is not running, the harness launches it automatically and retries.\n\nIf Chrome is running but remote debugging is not enabled, the harness opens:\n\n```text\nchrome://inspect/#remote-debugging\n```\n\nOn macOS, when Chrome asks for remote-debugging permission, run:\n\n```text\nbrowser-use mac-approve\n```\n\nContinue when it returns `ready`; otherwise follow its printed instruction.\n\n## Remote Browsers\n\nUse Browser Use Cloud for headless servers, parallel sub-agents, or isolated\nwork. Each cloud browser is a fresh, isolated managed Chrome instance. Prefer\none when:\n\n- The user wants multiple concurrent tasks. One cloud browser per task avoids\n tabs and focus fighting in a shared local browser.\n- Captchas or blocking are likely. Cloud browsers use clean managed IPs and\n stealth settings, keeping the user's own IP and browser out of the task.\n\nAuthenticate once:\n\n```bash\nbrowser-use auth login --device-code\n```\n\nOr import a key safely:\n\n```bash\nprintf '%s' \"$BROWSER_USE_API_KEY\" | browser-use auth login --api-key-stdin\n```\n\nPick a short made-up name; `r7k2` below is only a placeholder:\n\n```bash\nbrowser-use <<'PY'\nstart_remote_daemon(\"r7k2\")\nPY\n\nBU_NAME=r7k2 browser-use <<'PY'\nnew_tab(\"https://example.com\")\nprint(page_info())\nPY\n```\n\nWhen the task is done and a cloud browser is still running, ask directly:\n\"Should I close this browser now?\" If yes, run `stop_remote_daemon(name)`.\nRemote daemons bill until they stop or time out. Always use the same `BU_NAME`\nafter starting a named remote daemon.\n\nCloud profile cookie sync reference:\nhttps://github.com/browser-use/browser-harness/blob/main/interaction-skills/profile-sync.md.\n\n## Page Workflow\n\n- Prefer the accessibility tree over screenshots for finding elements:\n `cdp(\"Accessibility.getFullAXTree\")[\"nodes\"]` includes roles, names, and\n `backendDOMNodeId` values. Filter in Python before printing because it can\n contain thousands of nodes.\n- Resolve an accessibility node's box with\n `cdp(\"DOM.getBoxModel\", backendNodeId=n)`, click its center with\n `click_at_xy(x, y)`, and verify the result with targeted `js(...)` or\n `page_info()` output.\n- Fall back to raw HTML through `js(...)` only when the accessibility tree lacks\n the element, such as canvas or unusual widgets. Use screenshots when layout\n or imagery matters.\n- After navigation, call `wait_for_load()`.\n- If the current tab is stale or internal, call `ensure_real_tab()`.\n- Use `js(...)` for DOM inspection or extraction when coordinates are the wrong\n tool.\n- Stop at login walls. You may use available SSO when Chrome is already signed\n in, but stop for passwords, MFA, consent, or an ambiguous account choice.\n- Raw CDP is available with `cdp(\"Domain.method\", ...)`.\n\n## Recordings and Videos\n\nFresh installs do not record. Users can enable local background traces:\n\n```bash\nbrowser-use recordings enable\nbrowser-use recordings disable\nbrowser-use recordings\n```\n\n`BH_RECORD=1` or `BH_RECORD=0` overrides the preference for one process. A\nrequest to record, show, demo, or make a video opts in that task; significant\nwork alone does not.\n\nBefore browser work, call `start_recording(name, title=...)`, retain its exact\nreturned directory, and call `stop_recording()` after verification. Never\nreplace that path with `recordings --latest`. For a request made after the task,\nuse:\n\n```bash\nbrowser-use recordings --latest\n```\n\nUse it only if timestamps and pages match. Otherwise say the work was not\ncaptured. Never reenact completed work. For a video, follow\nhttps://github.com/browser-use/browser-harness/blob/main/interaction-skills/make-video.md.\n\n## Interaction Skills\n\nIf a browser mechanic is blocking progress, check the focused guides at\nhttps://github.com/browser-use/browser-harness/tree/main/interaction-skills.\nThey cover cookies, cross-origin iframes, dialogs, downloads, drag and drop,\ndropdowns, uploads, scrolling, screenshots, tabs, and viewports.\n\n## Design Constraints\n\n- Coordinate clicks are the default. CDP mouse events pass through iframe,\n shadow DOM, and cross-origin boundaries at the compositor level.\n- Keep the connection model simple: use the default daemon, `BU_NAME`,\n `BU_CDP_URL`, `BU_CDP_WS`, or `start_remote_daemon(...)`.\n- Keep core helpers short. Put task-specific helper additions in\n `$BH_AGENT_WORKSPACE/agent_helpers.py`.\n\n## Gotchas\n\n- `chrome://inspect/#remote-debugging` must be enabled for local Chrome control.\n- On macOS, if Chrome shows an \"Allow remote debugging?\" popup, run\n `browser-use mac-approve`. Do not poll in a loop because the daemon holds one\n connection.\n- Omnibox popups are not real work tabs.\n- CDP target order is not Chrome's visible tab-strip order.\n- `BU_CDP_URL` is an HTTP DevTools endpoint; the daemon resolves it to a\n WebSocket.\n- Ask before leaving cloud browsers running. Stop them with\n `stop_remote_daemon(name)` or `PATCH /browsers/{id} {\"action\":\"stop\"}`.\n\n## Domain Skills\n\nThis section applies only when `BH_DOMAIN_SKILLS=1`.\n\nWhen enabled, search `$BH_AGENT_WORKSPACE/domain-skills//` before\ninventing an approach. `goto_url(...)` returns up to 10 skill filenames for the\nnavigated host.", + "content": "# Browser Use CLI 3.0\n\nUse Browser Use through OpenHands' terminal instead of the built-in structured\nbrowser tools while this skill is active. Browser Use keeps the browser API out\nof the model's tool schema and exposes direct CDP helpers through a Python CLI.\n\n## Setup\n\nInstall or upgrade the current Browser Use CLI in an isolated `uv` tool\nenvironment:\n\n```bash\nuv tool install --python 3.12 --upgrade --force browser-use\nBU_CLI=\"$(uv tool dir --bin)/browser-use\"\nenv -u PYTHONPATH -u PYTHONHOME \"$BU_CLI\" --help\n```\n\nUse the exact `BU_CLI` path and clean child environment for every invocation.\nThis prevents a project virtualenv's older `browser-use` executable or Python\npackages from shadowing the isolated CLI. For setup or connection problems, run\n`env -u PYTHONPATH -u PYTHONHOME \"$BU_CLI\" --doctor` and read\nhttps://github.com/browser-use/browser-harness/blob/main/install.md.\n\n## When Not to Use\n\nA basic fetch of public information needs no browser. If a plain HTTP request\ncan read a public page, API, or documentation, use `curl` or another fetch tool.\nUse Browser Use when the task needs interaction, the user's logged-in session,\nJavaScript rendering, or a bot-protected page. If a direct fetch fails or\nreturns a shell page, escalate to the browser.\n\nDomain skills are off by default. Set `BH_DOMAIN_SKILLS=1` to enable them; see\nthe bottom section.\n\nIf `BH_DOMAIN_SKILLS=1` and the task is site-specific, read every file in the\nmatching `$BH_AGENT_WORKSPACE/domain-skills//` directory before inventing\nan approach.\n\n## Usage\n\n```bash\nBU_CLI=\"$(uv tool dir --bin)/browser-use\"\nenv -u PYTHONPATH -u PYTHONHOME \"$BU_CLI\" <<'PY'\nprint(page_info())\nPY\n```\n\n- Invoke through `BU_CLI` with the clean environment shown above. Use heredocs\n for multi-line commands.\n- Helpers are pre-imported. The CLI starts its daemon before execution.\n- First navigation is `new_tab(url)`, not `goto_url(url)`.\n- The normal local flow attaches to a running Chrome or Chromium CDP endpoint.\n\n## Local Chrome\n\nIf the daemon cannot connect, run diagnostics:\n\n```bash\nBU_CLI=\"$(uv tool dir --bin)/browser-use\"\nenv -u PYTHONPATH -u PYTHONHOME \"$BU_CLI\" --doctor\n```\n\nIf Chrome is not running, the harness launches it automatically and retries.\n\nIf Chrome is running but remote debugging is not enabled, the harness opens:\n\n```text\nchrome://inspect/#remote-debugging\n```\n\nOn macOS, when Chrome asks for remote-debugging permission, run:\n\n```text\nenv -u PYTHONPATH -u PYTHONHOME \"$BU_CLI\" mac-approve\n```\n\nContinue when it returns `ready`; otherwise follow its printed instruction.\n\n## Remote Browsers\n\nUse Browser Use Cloud for headless servers, parallel sub-agents, or isolated\nwork. Each cloud browser is a fresh, isolated managed Chrome instance. Prefer\none when:\n\n- The user wants multiple concurrent tasks. One cloud browser per task avoids\n tabs and focus fighting in a shared local browser.\n- Captchas or blocking are likely. Cloud browsers use clean managed IPs and\n stealth settings, keeping the user's own IP and browser out of the task.\n\nAuthenticate once:\n\n```bash\nBU_CLI=\"$(uv tool dir --bin)/browser-use\"\nenv -u PYTHONPATH -u PYTHONHOME \"$BU_CLI\" auth login --device-code\n```\n\nOr import a key safely:\n\n```bash\nBU_CLI=\"$(uv tool dir --bin)/browser-use\"\nprintf '%s' \"$BROWSER_USE_API_KEY\" | \\\n env -u PYTHONPATH -u PYTHONHOME \"$BU_CLI\" auth login --api-key-stdin\n```\n\nPick a short made-up name; `r7k2` below is only a placeholder:\n\n```bash\nBU_CLI=\"$(uv tool dir --bin)/browser-use\"\nenv -u PYTHONPATH -u PYTHONHOME \"$BU_CLI\" <<'PY'\nstart_remote_daemon(\"r7k2\")\nPY\n\nBU_NAME=r7k2 env -u PYTHONPATH -u PYTHONHOME \"$BU_CLI\" <<'PY'\nnew_tab(\"https://example.com\")\nprint(page_info())\nPY\n```\n\nWhen the task is done and a cloud browser is still running, ask directly:\n\"Should I close this browser now?\" If yes, run `stop_remote_daemon(name)`.\nRemote daemons bill until they stop or time out. Always use the same `BU_NAME`\nafter starting a named remote daemon.\n\nCloud profile cookie sync reference:\nhttps://github.com/browser-use/browser-harness/blob/main/interaction-skills/profile-sync.md.\n\n## Page Workflow\n\n- Prefer the accessibility tree over screenshots for finding elements:\n `cdp(\"Accessibility.getFullAXTree\")[\"nodes\"]` includes roles, names, and\n `backendDOMNodeId` values. Filter in Python before printing because it can\n contain thousands of nodes.\n- Resolve an accessibility node's box with\n `cdp(\"DOM.getBoxModel\", backendNodeId=n)`, click its center with\n `click_at_xy(x, y)`, and verify the result with targeted `js(...)` or\n `page_info()` output.\n- Fall back to raw HTML through `js(...)` only when the accessibility tree lacks\n the element, such as canvas or unusual widgets. Use screenshots when layout\n or imagery matters.\n- After navigation, call `wait_for_load()`.\n- If the current tab is stale or internal, call `ensure_real_tab()`.\n- Use `js(...)` for DOM inspection or extraction when coordinates are the wrong\n tool.\n- Stop at login walls. You may use available SSO when Chrome is already signed\n in, but stop for passwords, MFA, consent, or an ambiguous account choice.\n- Raw CDP is available with `cdp(\"Domain.method\", ...)`.\n\n## Recordings and Videos\n\nFresh installs do not record. Users can enable local background traces:\n\n```bash\nBU_CLI=\"$(uv tool dir --bin)/browser-use\"\nenv -u PYTHONPATH -u PYTHONHOME \"$BU_CLI\" recordings enable\nenv -u PYTHONPATH -u PYTHONHOME \"$BU_CLI\" recordings disable\nenv -u PYTHONPATH -u PYTHONHOME \"$BU_CLI\" recordings\n```\n\n`BH_RECORD=1` or `BH_RECORD=0` overrides the preference for one process. A\nrequest to record, show, demo, or make a video opts in that task; significant\nwork alone does not.\n\nBefore browser work, call `start_recording(name, title=...)`, retain its exact\nreturned directory, and call `stop_recording()` after verification. Never\nreplace that path with `recordings --latest`. For a request made after the task,\nuse:\n\n```bash\nBU_CLI=\"$(uv tool dir --bin)/browser-use\"\nenv -u PYTHONPATH -u PYTHONHOME \"$BU_CLI\" recordings --latest\n```\n\nUse it only if timestamps and pages match. Otherwise say the work was not\ncaptured. Never reenact completed work. For a video, follow\nhttps://github.com/browser-use/browser-harness/blob/main/interaction-skills/make-video.md.\n\n## Interaction Skills\n\nIf a browser mechanic is blocking progress, check the focused guides at\nhttps://github.com/browser-use/browser-harness/tree/main/interaction-skills.\nThey cover cookies, cross-origin iframes, dialogs, downloads, drag and drop,\ndropdowns, uploads, scrolling, screenshots, tabs, and viewports.\n\n## Design Constraints\n\n- Coordinate clicks are the default. CDP mouse events pass through iframe,\n shadow DOM, and cross-origin boundaries at the compositor level.\n- Keep the connection model simple: use the default daemon, `BU_NAME`,\n `BU_CDP_URL`, `BU_CDP_WS`, or `start_remote_daemon(...)`.\n- Keep core helpers short. Put task-specific helper additions in\n `$BH_AGENT_WORKSPACE/agent_helpers.py`.\n\n## Gotchas\n\n- `chrome://inspect/#remote-debugging` must be enabled for local Chrome control.\n- On macOS, if Chrome shows an \"Allow remote debugging?\" popup, run\n `browser-use mac-approve`. Do not poll in a loop because the daemon holds one\n connection.\n- Omnibox popups are not real work tabs.\n- CDP target order is not Chrome's visible tab-strip order.\n- `BU_CDP_URL` is an HTTP DevTools endpoint; the daemon resolves it to a\n WebSocket.\n- Ask before leaving cloud browsers running. Stop them with\n `stop_remote_daemon(name)` or `PATCH /browsers/{id} {\"action\":\"stop\"}`.\n\n## Domain Skills\n\nThis section applies only when `BH_DOMAIN_SKILLS=1`.\n\nWhen enabled, search `$BH_AGENT_WORKSPACE/domain-skills//` before\ninventing an approach. `goto_url(...)` returns up to 10 skill filenames for the\nnavigated host.", "category": "environment" }, {