From 7464ecd31f6e1b7bdc7849c93d377b7e98f3fd17 Mon Sep 17 00:00:00 2001 From: arelchan <204152633+arelchan@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:53:06 +0800 Subject: [PATCH] fix(*): resolve releases without spending the github api quota `raven upgrade` and both installers resolved the latest release through api.github.com with no credentials. GitHub meters unauthenticated API requests at 60 per hour per IP address, so every machine behind one office NAT draws down a single shared counter: once it is spent, upgrading and installing both fail with a 403 that the CLI reported as "Check your network and try again", sending users to diagnose the wrong thing. Resolve the release from the plain github.com redirect instead (releases/latest -> releases/tag/vX.Y.Z), which is not the REST API and has no such per-IP quota. Asset names follow from the tag, and the payload parser already required the wheel URL to be exactly the one the tag implies, so nothing is lost. The API stays as a fallback and now sends GITHUB_TOKEN / GH_TOKEN when either is set, which moves a developer or CI run onto its own 5000-per-hour quota. An exhausted quota is now identified from x-ratelimit-remaining and reported with the reset time from x-ratelimit-reset, and UpgradeError carries an optional hint so that message replaces the generic network advice instead of being appended to it. Both installers say the same thing and point at GITHUB_TOKEN and RAVEN_WHEEL_URL. Co-authored-by: Claude (claude-opus-5) --- install.ps1 | 36 ++++++- install.sh | 33 +++++- raven/cli/upgrade_commands.py | 120 ++++++++++++++++++--- tests/test_cli_upgrade_commands.py | 161 ++++++++++++++++++++++++++++- 4 files changed, 332 insertions(+), 18 deletions(-) diff --git a/install.ps1 b/install.ps1 index b54b5deb..12ea22f7 100644 --- a/install.ps1 +++ b/install.ps1 @@ -191,10 +191,44 @@ function Ensure-Node { } } +function Resolve-RavenWheelFromTag { + # Prefer the plain github.com redirect (releases/latest -> releases/tag/vX.Y.Z) + # over the REST API: the redirect is not the API, so it does not spend the + # 60-requests-per-hour unauthenticated quota GitHub meters per IP address -- + # one counter shared by everyone behind the same address. Release asset names + # follow from the tag. Any failure returns $null so the caller falls back to + # the API rather than making a piped install worse than before. + $latest = "https://github.com/EverMind-AI/Raven/releases/latest" + $location = $null + try { + $response = Invoke-WebRequest $latest -MaximumRedirection 0 -UseBasicParsing -ErrorAction SilentlyContinue + if ($response) { $location = $response.Headers["Location"] } + } catch { + # Windows PowerShell 5.1 raises on a 3xx instead of returning it. + $failed = $_.Exception.Response + if ($failed) { $location = $failed.Headers["Location"] } + } + if ($location -is [array]) { $location = $location[0] } + if (-not $location) { return $null } + if ($location -notmatch '^https://github\.com/EverMind-AI/Raven/releases/tag/(v\d+\.\d+\.\d+)$') { return $null } + $tag = $Matches[1] + $version = $tag.TrimStart("v") + return "https://github.com/EverMind-AI/Raven/releases/download/$tag/raven-$version-py3-none-any.whl" +} + function Resolve-RavenWheel { if ($env:RAVEN_WHEEL_URL) { return $env:RAVEN_WHEEL_URL } Write-Info "Resolving the latest Raven release from GitHub..." - $release = Invoke-RestMethod "https://api.github.com/repos/EverMind-AI/Raven/releases/latest" -Headers @{ "User-Agent" = "raven-installer" } + $fromTag = Resolve-RavenWheelFromTag + if ($fromTag) { return $fromTag } + $headers = @{ "User-Agent" = "raven-installer" } + $token = if ($env:GITHUB_TOKEN) { $env:GITHUB_TOKEN } elseif ($env:GH_TOKEN) { $env:GH_TOKEN } else { $null } + if ($token) { $headers["Authorization"] = "Bearer $token" } + try { + $release = Invoke-RestMethod "https://api.github.com/repos/EverMind-AI/Raven/releases/latest" -Headers $headers + } catch { + Fail ("Could not resolve the latest Raven release from GitHub: " + $_.Exception.Message + " Its unauthenticated API quota (60 per hour, metered per IP address) may be spent: set GITHUB_TOKEN to use your own quota, or set RAVEN_WHEEL_URL to a wheel URL.") + } $asset = $release.assets | Where-Object { $_.browser_download_url -match "/raven-[^/]+\.whl$" } | Select-Object -First 1 if (-not $asset) { Fail "Could not resolve the latest Raven release wheel from GitHub. Set RAVEN_WHEEL_URL to a wheel URL." diff --git a/install.sh b/install.sh index 048ec4b0..99bc92a5 100755 --- a/install.sh +++ b/install.sh @@ -146,6 +146,34 @@ ensure_node() { } # --- 3. install raven ------------------------------------------------------ +RELEASES_BASE="https://github.com/EverMind-AI/Raven/releases" +LATEST_RELEASE_API="https://api.github.com/repos/EverMind-AI/Raven/releases/latest" + +# Print the latest release wheel URL, or nothing when it cannot be resolved. +# +# Prefers the plain github.com redirect (releases/latest -> releases/tag/vX.Y.Z) +# over the REST API: the redirect is not the API, so it does not spend the +# 60-requests-per-hour unauthenticated quota that GitHub meters per IP address +# -- one counter shared by everyone behind an office NAT, which is why installs +# start failing in a busy network. Release asset names follow from the tag, so +# the tag is all this needs. The API stays as a fallback, authenticated when a +# token is in the environment (5000 per hour, metered per account). +resolve_wheel_url() { + tag="$(curl -fsS -o /dev/null -w '%{redirect_url}' "$RELEASES_BASE/latest" 2>/dev/null \ + | sed -n "s#^$RELEASES_BASE/tag/\(v[0-9][0-9.]*\)\$#\1#p")" + if [ -n "$tag" ]; then + printf '%s/download/%s/raven-%s-py3-none-any.whl\n' "$RELEASES_BASE" "$tag" "${tag#v}" + return 0 + fi + token="${GITHUB_TOKEN:-${GH_TOKEN:-}}" + if [ -n "$token" ]; then + body="$(curl -fsSL -H "Authorization: Bearer $token" "$LATEST_RELEASE_API" 2>/dev/null || true)" + else + body="$(curl -fsSL "$LATEST_RELEASE_API" 2>/dev/null || true)" + fi + printf '%s' "$body" | grep -oE 'https://[^"]*/raven-[^"]*\.whl' | head -n1 +} + install_raven() { # Local mode: run from a raven source checkout -> editable install of the # working tree (what a developer wants). Otherwise install from git. @@ -189,10 +217,9 @@ install_raven() { wheel_url="${RAVEN_WHEEL_URL:-}" if [ -z "$wheel_url" ]; then info "Resolving the latest raven release from GitHub..." - wheel_url="$(curl -fsSL "https://api.github.com/repos/EverMind-AI/raven/releases/latest" 2>/dev/null \ - | grep -oE 'https://[^"]*/raven-[^"]*\.whl' | head -n1)" + wheel_url="$(resolve_wheel_url)" fi - [ -n "$wheel_url" ] || die "Could not resolve the latest raven release wheel from GitHub (check network, or set RAVEN_WHEEL_URL to a wheel URL)." + [ -n "$wheel_url" ] || die "Could not resolve the latest raven release wheel from GitHub. GitHub may be unreachable, or its unauthenticated API quota (60 per hour, metered per IP address) may be spent: set GITHUB_TOKEN to use your own quota, or set RAVEN_WHEEL_URL to a wheel URL." # Derive the locked-constraints URL from the wheel URL (same release dir) so # the constraints always match the wheel being installed, including when # RAVEN_WHEEL_URL pins an older wheel. Missing asset / download failure -> diff --git a/raven/cli/upgrade_commands.py b/raven/cli/upgrade_commands.py index c6fe34e2..d29d4e5e 100644 --- a/raven/cli/upgrade_commands.py +++ b/raven/cli/upgrade_commands.py @@ -9,6 +9,7 @@ import sys import tomllib from dataclasses import dataclass +from datetime import datetime from importlib import metadata from pathlib import Path from urllib.parse import urlparse @@ -18,12 +19,25 @@ from rich.console import Console LATEST_RELEASE_API = "https://api.github.com/repos/EverMind-AI/Raven/releases/latest" +# Plain web redirect to the newest non-draft, non-prerelease tag. Not the REST +# API, so it does not spend the 60-per-hour unauthenticated API quota that +# GitHub meters per IP address (one shared counter for everyone behind an office +# NAT). Preferred over LATEST_RELEASE_API for that reason alone. +LATEST_RELEASE_URL = "https://github.com/EverMind-AI/Raven/releases/latest" +RELEASE_DOWNLOAD_PREFIX = "https://github.com/EverMind-AI/Raven/releases/download" _VERSION_RE = re.compile(r"^v?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$") +_TAG_PATH_RE = re.compile(r"^/EverMind-AI/Raven/releases/tag/(v(?:0|[1-9][0-9]*)(?:\.(?:0|[1-9][0-9]*)){2})$") +_REDIRECT_CODES = frozenset({301, 302, 303, 307, 308}) console = Console() class UpgradeError(RuntimeError): - pass + """An upgrade failure. ``hint`` replaces the generic remediation line when + the cause is understood well enough to say something more useful.""" + + def __init__(self, message: str, *, hint: str | None = None) -> None: + super().__init__(message) + self.hint = hint @dataclass(frozen=True) @@ -231,20 +245,101 @@ def _parse_release_payload(payload: object) -> ReleaseInfo: return ReleaseInfo(version=version, wheel_url=wheel_url) -def _fetch_latest_release(client: httpx.Client | None = None) -> ReleaseInfo: +def _github_token() -> str | None: + for name in ("GITHUB_TOKEN", "GH_TOKEN"): + token = os.environ.get(name, "").strip() + if token: + return token + return None + + +def _api_headers() -> dict[str, str]: headers = { "Accept": "application/vnd.github+json", "User-Agent": f"raven/{_current_version()}", "X-GitHub-Api-Version": "2022-11-28", } + token = _github_token() + if token: + headers["Authorization"] = f"Bearer {token}" + return headers + + +def _release_from_tag(tag_name: str) -> ReleaseInfo: + """Build the release info a tag implies. + + Release asset names are deterministic, and ``_parse_release_payload`` + already rejects any wheel URL that is not exactly this one, so resolving a + tag is equivalent to reading the API payload for it. + """ + version = ".".join(str(part) for part in _version_key(tag_name)) + wheel_name = f"raven-{version}-py3-none-any.whl" + return ReleaseInfo(version=version, wheel_url=f"{RELEASE_DOWNLOAD_PREFIX}/v{version}/{wheel_name}") + + +def _fetch_latest_release_via_redirect(client: httpx.Client) -> ReleaseInfo: + response = client.get( + LATEST_RELEASE_URL, + headers={"User-Agent": f"raven/{_current_version()}"}, + follow_redirects=False, + ) + if response.status_code not in _REDIRECT_CODES: + raise UpgradeError(f"GitHub did not redirect to a Raven release tag (HTTP {response.status_code})") + location = response.headers.get("location", "") + match = _TAG_PATH_RE.match(urlparse(location).path) if location else None + if match is None: + raise UpgradeError(f"Unexpected latest-release redirect target: {location or '(none)'}") + return _release_from_tag(match.group(1)) + + +def _rate_limit_error(response: httpx.Response) -> UpgradeError | None: + """Turn an exhausted-quota API response into an error that says so. + + GitHub answers a spent quota with 403 (or 429) plus + ``x-ratelimit-remaining: 0``; without reading those headers the failure is + indistinguishable from a network fault, which is what the generic + remediation line used to claim it was. + """ + if response.status_code not in (403, 429) or response.headers.get("x-ratelimit-remaining") != "0": + return None + limit = response.headers.get("x-ratelimit-limit", "?") + reset = response.headers.get("x-ratelimit-reset", "") + when = "" + if reset.isdigit(): + when = f", resetting at {datetime.fromtimestamp(int(reset)).strftime('%H:%M:%S')}" + hint = "Wait for the reset, or rerun the official installer, which resolves releases without the API." + if _github_token() is None: + hint = ( + "GitHub meters unauthenticated requests per IP address, so anyone sharing yours " + "spends the same quota. Set GITHUB_TOKEN to use your own, or wait for the reset." + ) + return UpgradeError( + f"GitHub API quota of {limit} requests per hour is exhausted for this IP address{when}", + hint=hint, + ) + + +def _fetch_latest_release_via_api(client: httpx.Client) -> ReleaseInfo: + response = client.get(LATEST_RELEASE_API, headers=_api_headers()) + quota_error = _rate_limit_error(response) + if quota_error is not None: + raise quota_error + response.raise_for_status() + return _parse_release_payload(response.json()) + + +def _fetch_latest_release_from(client: httpx.Client) -> ReleaseInfo: + try: + return _fetch_latest_release_via_redirect(client) + except (UpgradeError, httpx.HTTPError): + return _fetch_latest_release_via_api(client) + + +def _fetch_latest_release(client: httpx.Client | None = None) -> ReleaseInfo: if client is not None: - response = client.get(LATEST_RELEASE_API, headers=headers) - response.raise_for_status() - return _parse_release_payload(response.json()) - with httpx.Client(timeout=10.0, follow_redirects=True) as owned_client: - response = owned_client.get(LATEST_RELEASE_API, headers=headers) - response.raise_for_status() - return _parse_release_payload(response.json()) + return _fetch_latest_release_from(client) + with httpx.Client(timeout=10.0) as owned_client: + return _fetch_latest_release_from(owned_client) def _direct_url_data() -> dict[str, object] | None: @@ -456,9 +551,8 @@ def upgrade( ValueError, metadata.PackageNotFoundError, ) as exc: - console.print( - f"[red]Unable to upgrade Raven:[/red] {exc}. " - "Check your network and try again; if the problem persists, " - "rerun the official installer." + hint = getattr(exc, "hint", None) or ( + "Check your network and try again; if the problem persists, rerun the official installer." ) + console.print(f"[red]Unable to upgrade Raven:[/red] {exc}. {hint}") raise typer.Exit(1) from exc diff --git a/tests/test_cli_upgrade_commands.py b/tests/test_cli_upgrade_commands.py index 452ddd5d..66218200 100644 --- a/tests/test_cli_upgrade_commands.py +++ b/tests/test_cli_upgrade_commands.py @@ -180,14 +180,85 @@ def test_parse_release_payload_rejects_untrusted_wheel_urls(wheel_url: str) -> N upgrade_commands._parse_release_payload(_release_payload(assets=assets)) -def test_fetch_latest_release_uses_github_api_contract(monkeypatch: pytest.MonkeyPatch) -> None: +def _tag_redirect(version: str = "0.1.4") -> httpx.Response: + return httpx.Response( + 302, + headers={"location": f"https://github.com/EverMind-AI/Raven/releases/tag/v{version}"}, + ) + + +def _quota_response(remaining: str = "0", reset: str = "1785243845") -> httpx.Response: + return httpx.Response( + 403, + headers={ + "x-ratelimit-limit": "60", + "x-ratelimit-remaining": remaining, + "x-ratelimit-reset": reset, + }, + json={"message": "API rate limit exceeded"}, + ) + + +def test_release_from_tag_builds_the_url_the_payload_parser_accepts() -> None: + from_tag = upgrade_commands._release_from_tag("v0.1.4") + + assert from_tag == upgrade_commands._parse_release_payload(_release_payload()) + + +def test_fetch_latest_release_prefers_the_redirect_over_the_api(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(upgrade_commands, "_current_version", lambda: "0.1.3") + seen: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(str(request.url)) + assert request.headers["User-Agent"] == "raven/0.1.3" + return _tag_redirect() + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + release = upgrade_commands._fetch_latest_release(client) + + assert release == upgrade_commands.ReleaseInfo(version="0.1.4", wheel_url=WHEEL_URL) + assert seen == [upgrade_commands.LATEST_RELEASE_URL] + + +def test_fetch_latest_release_does_not_follow_the_redirect(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(upgrade_commands, "_current_version", lambda: "0.1.3") def handler(request: httpx.Request) -> httpx.Response: + if str(request.url) == upgrade_commands.LATEST_RELEASE_URL: + return _tag_redirect() + raise AssertionError(f"followed the redirect to {request.url}") + + with httpx.Client(transport=httpx.MockTransport(handler), follow_redirects=True) as client: + release = upgrade_commands._fetch_latest_release(client) + + assert release.version == "0.1.4" + + +@pytest.mark.parametrize( + "response", + [ + pytest.param(httpx.Response(200), id="no-redirect"), + pytest.param( + httpx.Response(302, headers={"location": "https://github.com/login"}), + id="unexpected-target", + ), + pytest.param(httpx.Response(302), id="redirect-without-location"), + ], +) +def test_fetch_latest_release_falls_back_to_the_api(response: httpx.Response, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(upgrade_commands, "_current_version", lambda: "0.1.3") + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + monkeypatch.delenv("GH_TOKEN", raising=False) + + def handler(request: httpx.Request) -> httpx.Response: + if str(request.url) == upgrade_commands.LATEST_RELEASE_URL: + return response assert str(request.url) == upgrade_commands.LATEST_RELEASE_API assert request.headers["Accept"] == "application/vnd.github+json" assert request.headers["User-Agent"] == "raven/0.1.3" assert request.headers["X-GitHub-Api-Version"] == "2022-11-28" + assert "Authorization" not in request.headers return httpx.Response(200, json=_release_payload()) with httpx.Client(transport=httpx.MockTransport(handler)) as client: @@ -196,6 +267,76 @@ def handler(request: httpx.Request) -> httpx.Response: assert release == upgrade_commands.ReleaseInfo(version="0.1.4", wheel_url=WHEEL_URL) +@pytest.mark.parametrize("env_var", ["GITHUB_TOKEN", "GH_TOKEN"]) +def test_fetch_latest_release_authenticates_the_api_when_a_token_is_set( + env_var: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + monkeypatch.delenv("GH_TOKEN", raising=False) + monkeypatch.setenv(env_var, " gho_secret ") + + def handler(request: httpx.Request) -> httpx.Response: + if str(request.url) == upgrade_commands.LATEST_RELEASE_URL: + return httpx.Response(500) + assert request.headers["Authorization"] == "Bearer gho_secret" + return httpx.Response(200, json=_release_payload()) + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + release = upgrade_commands._fetch_latest_release(client) + + assert release.version == "0.1.4" + + +def test_fetch_latest_release_reports_an_exhausted_api_quota(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + monkeypatch.delenv("GH_TOKEN", raising=False) + + def handler(request: httpx.Request) -> httpx.Response: + if str(request.url) == upgrade_commands.LATEST_RELEASE_URL: + return httpx.Response(500) + return _quota_response() + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + with pytest.raises(upgrade_commands.UpgradeError) as excinfo: + upgrade_commands._fetch_latest_release(client) + + message = str(excinfo.value) + assert "quota of 60 requests per hour is exhausted" in message + assert "for this IP address" in message + assert "resetting at" in message + assert excinfo.value.hint is not None + assert "GITHUB_TOKEN" in excinfo.value.hint + + +def test_fetch_latest_release_quota_hint_skips_the_token_advice_when_authenticated( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("GITHUB_TOKEN", "gho_secret") + + def handler(request: httpx.Request) -> httpx.Response: + if str(request.url) == upgrade_commands.LATEST_RELEASE_URL: + return httpx.Response(500) + return _quota_response() + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + with pytest.raises(upgrade_commands.UpgradeError) as excinfo: + upgrade_commands._fetch_latest_release(client) + + assert excinfo.value.hint is not None + assert "GITHUB_TOKEN" not in excinfo.value.hint + + +def test_fetch_latest_release_keeps_a_plain_403_a_status_error(monkeypatch: pytest.MonkeyPatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if str(request.url) == upgrade_commands.LATEST_RELEASE_URL: + return httpx.Response(500) + return _quota_response(remaining="7") + + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + with pytest.raises(httpx.HTTPStatusError): + upgrade_commands._fetch_latest_release(client) + + def test_fetch_latest_release_propagates_timeout() -> None: def handler(request: httpx.Request) -> httpx.Response: raise httpx.ReadTimeout("timed out", request=request) @@ -822,6 +963,24 @@ def test_upgrade_check_reports_available_without_install(monkeypatch: pytest.Mon handoff.assert_not_called() +def test_upgrade_replaces_the_network_advice_with_the_error_hint(monkeypatch: pytest.MonkeyPatch) -> None: + def quota_exhausted() -> upgrade_commands.ReleaseInfo: + raise upgrade_commands.UpgradeError( + "GitHub API quota of 60 requests per hour is exhausted for this IP address", + hint="Set GITHUB_TOKEN to use your own, or wait for the reset.", + ) + + monkeypatch.setattr(upgrade_commands, "_fetch_latest_release", quota_exhausted) + + result = runner.invoke(app, ["upgrade"]) + + output = " ".join(result.stdout.split()) + assert result.exit_code == 1 + assert "quota of 60 requests per hour is exhausted" in output + assert "Set GITHUB_TOKEN" in output + assert "Check your network" not in output + + def test_upgrade_reports_current_release_as_up_to_date(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(upgrade_commands, "_current_version", lambda: "0.1.4") monkeypatch.setattr(