From f3ae332800794684c5cc7543bf1618a08148d476 Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 18:03:06 +0800 Subject: [PATCH 01/78] refactor(providers): give every provider decision one home Four decisions about providers were implemented outside raven/providers/: the credential-gate exit, the price and context-window ladder, pin resolution, and the predicate for which wire accepts a cache dialect. Each had grown a second copy at the surface that needed it, and the copies had drifted, so the same provider answered the same question differently depending on which entry point asked. Each decision now has one module that owns it, and the surfaces call it: wire.py outbound route prefix and inbound storage form auth.py credential requirements as an AND-of-OR grammar pin.py which provider a stored model routes to rates.py price and context window, with an explicit ladder prompt_cache.py who accepts cache_control, and who refused it catalog.py model labels, from a snapshot pinned to a commit Behaviour this changes for a user: every surface that writes the default model now also decides its pin, so the model is served by the provider just configured rather than by whichever one a keyword matched; missing credentials are reported as the sentence naming what is missing instead of an exit; a provider that refuses cache_control is learned once and not asked again. Co-authored-by: Claude (claude-opus-5[1m]) --- CONTEXT.md | 77 +++ .../pinchbench/direct/raven_executor.py | 40 +- pyproject.toml | 2 + raven/agent/loop/main.py | 2 +- raven/cli/_helpers.py | 69 +- raven/cli/commands.py | 11 + raven/cli/onboard_commands.py | 93 ++- raven/cli/provider_commands.py | 129 +++- raven/cli/sentinel_commands.py | 6 +- raven/cli/status_commands.py | 10 +- raven/cli/tui_commands.py | 13 + raven/config/schema.py | 134 +++- raven/config/update.py | 11 +- raven/config/update_providers.py | 143 +++- raven/providers/auth.py | 275 ++++++++ raven/providers/azure_openai_provider.py | 20 +- raven/providers/base.py | 51 +- raven/providers/capabilities.py | 46 +- raven/providers/catalog.py | 179 +++++ raven/providers/common_models.py | 22 +- raven/providers/data/models_dev.json | 1 + raven/providers/litellm_provider.py | 156 +++-- .../model_catalog_cache.py | 2 +- raven/providers/openai_codex_provider.py | 12 +- raven/providers/pin.py | 113 ++++ raven/providers/prompt_cache.py | 210 ++++++ raven/providers/rates.py | 508 ++++++++++++++ raven/providers/registry.py | 18 - raven/providers/wire.py | 224 +++++++ raven/token_wise/cache_optimizer.py | 16 +- raven/token_wise/pricing.py | 493 +------------- raven/token_wise/system_and_tail_cache.py | 16 +- raven/tracing/semconv.py | 5 +- raven/tui_rpc/methods/config.py | 89 +-- raven/tui_rpc/methods/model.py | 87 ++- raven/tui_rpc/methods/session.py | 4 +- raven/tui_rpc/methods/setup.py | 35 +- raven/tui_rpc/models.py | 13 + scripts/refresh_models_dev_snapshot.py | 259 ++++++++ tests/conftest.py | 12 +- tests/data/wire_model_baseline.json | 199 ++++++ tests/integration/test_provider_real_llm.py | 269 ++++++++ tests/test_agent_loop_usage_sink.py | 14 +- tests/test_azure_openai_provider.py | 30 + tests/test_bedrock_stub.py | 33 +- tests/test_cli_helpers.py | 50 +- tests/test_cli_onboard_commands.py | 64 +- tests/test_cli_provider_commands.py | 192 +++++- tests/test_config_update_providers.py | 178 ++++- tests/test_provider_auth_method.py | 301 +++++++++ tests/test_provider_catalog.py | 250 +++++++ tests/test_provider_pin.py | 235 +++++++ tests/test_provider_prompt_cache.py | 546 +++++++++++++++ tests/test_provider_rates.py | 629 ++++++++++++++++++ tests/test_provider_resolution_invariants.py | 76 ++- tests/test_provider_stream_fallback.py | 190 +++++- tests/test_provider_wire_model.py | 236 +++++++ tests/test_read_file_image.py | 19 +- tests/test_token_wise_cache_optimizer.py | 52 +- tests/test_token_wise_pricing.py | 576 +--------------- tests/test_tui_rpc_config.py | 27 +- tests/test_tui_rpc_model.py | 73 +- tests/test_tui_rpc_setup.py | 32 +- ui-tui/rpc-schema/openrpc.json | 17 +- .../src/__tests__/gatewayTypesDrift.test.ts | 49 ++ ui-tui/src/__tests__/modelPicker.test.tsx | 41 ++ ui-tui/src/components/modelPicker.tsx | 14 +- ui-tui/src/gatewayTypes.ts | 7 + ui-tui/src/rpc/generated.ts | 16 + 69 files changed, 6509 insertions(+), 1512 deletions(-) create mode 100644 raven/providers/auth.py create mode 100644 raven/providers/catalog.py create mode 100644 raven/providers/data/models_dev.json rename raven/{token_wise => providers}/model_catalog_cache.py (97%) create mode 100644 raven/providers/pin.py create mode 100644 raven/providers/prompt_cache.py create mode 100644 raven/providers/rates.py create mode 100644 raven/providers/wire.py create mode 100644 scripts/refresh_models_dev_snapshot.py create mode 100644 tests/data/wire_model_baseline.json create mode 100644 tests/integration/test_provider_real_llm.py create mode 100644 tests/test_provider_auth_method.py create mode 100644 tests/test_provider_pin.py create mode 100644 tests/test_provider_prompt_cache.py create mode 100644 tests/test_provider_rates.py create mode 100644 tests/test_provider_wire_model.py create mode 100644 ui-tui/src/__tests__/gatewayTypesDrift.test.ts diff --git a/CONTEXT.md b/CONTEXT.md index eb16c695..47d3b03f 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -207,6 +207,83 @@ An LLM vendor adapter (`providers/`: Anthropic, OpenAI, Gemini, …), shared by agent loop and the Curator. _Avoid_: conflating provider (vendor) with model (a model name a provider serves) +A Provider is described along four independent axes -- identity, connection, routing, +and what its models can do -- each with its own home. Mixing them in one record is what +left per-model facts nowhere to live and per-provider facts stated in several places at +once. The terms below name the pieces those axes are built from; they are properties of +a model or of a connection, not four synonyms for Provider. + +**Model Ref**: +The canonical way a model is written down: `provider/model`, naming whoever serves it. +Usually that is the section it was configured under; where a Provider declares +`skip_prefixes` it may instead be the gateway already named in the id +(`openrouter/z-ai/glm-4.6` stored under `zai` keeps OpenRouter's name, because +OpenRouter is what serves it). Produced by `providers/wire.py::stored_model_id`, which +every surface that persists a choice goes through. +_Avoid_: "model id" for the stored form when the sent form is also in play — say Model +Ref or Wire Model. + +**Merge Key**: +The identity of a Model Ref for comparison and de-duplication — the provider and the +vendor's own id, spelling-folded. Two refs naming one model share a Merge Key whatever +spelling either was written in. + +**Wire Model**: +The form a Model Ref takes on the request: a LiteLLM route string, an Azure deployment +name, or a Codex slug. Derived, never stored, and derived in one place +(`providers/wire.py::wire_model`). +_Avoid_: treating the stored and sent forms as one string — they differ per provider. + +**Auth Method**: +One way of connecting to a Provider: what credential material it needs (as an AND of +OR-groups), how that material is obtained, where it is kept, and how it is verified. +A Provider may declare several and is usable when any one is satisfied. +`providers/auth.py::credential_status` answers "is this Provider usable", and is the +only place that may: seven surfaces once decided it independently and disagreed with +each other on the two configurations that made the rewrite necessary. +_Avoid_: "credential kind" for the whole shape — that names only the material. + +**Model Row**: +One model as a person reads it: a Model Ref plus a label and a description, tagged with +the source that supplied them. Display only — nothing shaping a request reads a Model +Row (`providers/catalog.py`). +_Avoid_: confusing it with what a model can *do*. Whether a request may carry +`cache_control` blocks is a Prompt Cache Breakpoint question, not a Model Row one. + +**Model Overlay**: +What a user states about a model no catalogue carries — a label and a description for a +self-hosted deployment. Beats the catalogue for the fields it sets. + +**Prompt Cache Breakpoint**: +An Anthropic-shaped `cache_control` marker placed on a request so the prefix before it is +cached. Whether one may be placed is **(wire x model family)**: the wire has to have +somewhere to carry the field (`ProviderSpec.supports_prompt_caching`, a property of the +API being spoken) *and* the model's vendor has to be the one that reads it. A gateway +accepting the field is not the same as its upstream honouring it -- OpenRouter carries it +for every model it fronts and forwards it to vendors that bill the prompt twice. +Decided once, in `providers/prompt_cache.py`, which every marker asks. +_Avoid_: reading LiteLLM's per-model `supports_prompt_caching`, which answers "does this +model cache at all" -- a different question, and the one that produced the doubled bill. + +**Token Rates**: +What a model costs per token, and separately how much context it holds. Both are facts +about a Provider's catalogue, so both are resolved in `providers/rates.py` rather than by +whoever is about to report a number. The two are deliberately sourced differently: rates +price a call after it happened, so the ladder may reach a community-maintained catalogue; +a context window sizes trimming and therefore shapes the *next* request, so only the +tables that also route may answer it, and unknown is answered with the caller's own +configured default. +_Avoid_: "pricing" for the resolution -- that names the arithmetic on top +(`token_wise/pricing.py`), which is a different module for a reason. + +**Provider Pin**: +`agents.defaults.provider`: an explicit override of the Provider a Model Ref names. +Every surface that changes the model rewrites it by one rule +(`providers/pin.py::resolve`), because a pin left behind routes the new model to the old +vendor with the old vendor's key. +_Avoid_: reading it as a provider *signal* -- a pinned name says which section to ask +about, never that the section holds credentials. + ### TUI-RPC **TUI-RPC**: diff --git a/benchmarks/pinchbench/direct/raven_executor.py b/benchmarks/pinchbench/direct/raven_executor.py index 654d21ff..b42405de 100644 --- a/benchmarks/pinchbench/direct/raven_executor.py +++ b/benchmarks/pinchbench/direct/raven_executor.py @@ -330,38 +330,20 @@ def _make_benchmark_provider(model: str, api_key: str, api_base: str, provider_n def _estimate_cost_usd(model: str, prompt_tokens: int, completion_tokens: int) -> float | None: - """Estimate USD cost using LiteLLM's pricing database with manual fallbacks. + """Estimate USD cost for one call, or None when no source prices the model. - Falls back to _fallback_pricing for models not yet in LiteLLM's DB. - Returns None if the model is unknown to both. + A thin wrapper on purpose. This carried its own two-tier copy of the + resolution -- LiteLLM, then a hand-written table -- while the shipped ladder + grew a live gateway table and a per-vendor catalogue between them, so a + benchmark could report a figure the product would not. There is one answer to + "what does this cost" and it lives with the provider. """ - # Manual fallback pricing ($/token) for models absent from LiteLLM's DB. - # Source: OpenRouter model pages (as of 2026-03). - _fallback_pricing: Dict[str, tuple[float, float]] = { - "z-ai/glm-4.5-air": (0.13e-6, 0.85e-6), # $0.13/$0.85 per 1M tokens - } - - try: - import litellm - - # LiteLLM expects "openrouter//" format for OpenRouter models. - or_model = f"openrouter/{model}" if not model.startswith("openrouter/") else model - prompt_cost, completion_cost = litellm.cost_per_token( - model=or_model, - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - ) - return prompt_cost + completion_cost - except Exception: - pass - - # Fallback for models not in LiteLLM DB - base_model = model.removeprefix("openrouter/") - if base_model in _fallback_pricing: - p_per_tok, c_per_tok = _fallback_pricing[base_model] - return p_per_tok * prompt_tokens + c_per_tok * completion_tokens + from raven.token_wise.pricing import estimate_cost_usd - return None + # Benchmarks name OpenRouter models bare; the ladder is keyed by stored ids, + # which name their provider. + stored = model if model.startswith("openrouter/") else f"openrouter/{model}" + return estimate_cost_usd(stored, prompt_tokens, completion_tokens) async def _run_turn_text(agent, message: str, *, session_key: str, chat_id: str) -> str: diff --git a/pyproject.toml b/pyproject.toml index c73b4803..8129685b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -147,6 +147,8 @@ include = [ "raven/templates/**/*.md", "raven/skills/**/*.md", "raven/skills/**/*.sh", + # Bundled model-label snapshot, so a fresh install labels models offline. + "raven/providers/data/*.json", # Tracing dashboard viewer (dependency-free Node server + static client). "raven/tracing/viewer/**/*.js", "raven/tracing/viewer/**/*.css", diff --git a/raven/agent/loop/main.py b/raven/agent/loop/main.py index 49cee9b9..afe8247d 100644 --- a/raven/agent/loop/main.py +++ b/raven/agent/loop/main.py @@ -45,10 +45,10 @@ from raven.memory_engine.consolidate.consolidator import MemoryConsolidator, MemoryStore from raven.providers.base import LLMProvider, LLMResponse, ToolCallRequest from raven.providers.capabilities import image_placeholder_text, supports_image_tool_result, vision_verdict +from raven.providers.rates import resolve_context_window from raven.sandbox import SandboxConfig, SandboxExecutor, SandboxInitError, build_executor from raven.session.manager import Session, SessionManager from raven.spine.turn import Origin -from raven.token_wise.pricing import resolve_context_window from raven.tracing import semconv, trace from raven.utils.helpers import estimate_prompt_tokens, is_image_part, is_inline_image diff --git a/raven/cli/_helpers.py b/raven/cli/_helpers.py index cd719878..f6571d26 100644 --- a/raven/cli/_helpers.py +++ b/raven/cli/_helpers.py @@ -65,32 +65,40 @@ def check_provider_credentials(config: Config) -> None: """Fail-fast when the configured provider is missing required credentials. Cheap (no litellm import), so it can run at startup even when the real - provider is built lazily. It branches on the same ``client`` that decides - which provider gets built: kept in sync by asking the same question, because - the version that compared names drifted the moment the factory stopped. + provider is built lazily. + + Raises ``MissingCredentialsError`` rather than printing and exiting: three entry + points call this, and only one of them is a terminal. Each renders the + failure in its own idiom -- the CLI as a red line and exit 1, the TUI as an + RPC error carrying the same sentence. + + What counts as configured is `providers.auth`, the same declaration routing + and `provider list` consult. Deciding it here as well is what produced three + verdicts on one config: a Gemini section holding only `api_key_list` read as + configured in `provider list` and refused to start, and Azure with a key and + no address was routed and displayed as configured yet rejected here. """ + from raven.providers.auth import MissingCredentialsError, credential_status + from raven.providers.registry import find_by_model, split_model_id + model = config.agents.defaults.model provider_name = config.get_provider_name(model) - p = config.get_provider(model) - - from raven.providers.registry import find_by_name - - spec = find_by_name(provider_name) - client = spec.client if spec else "" + if not provider_name: + # Routing found no configured section, so name the provider the model id + # points at rather than reporting on nothing. + spec = find_by_model(model) + provider_name = spec.name if spec else split_model_id(model)[0] + if not provider_name: + raise MissingCredentialsError( + "no provider configured", + # A command, not a config path: the old text pointed at + # ~/.raven/config.json, the layout the CLI exists to hide. + remedy="Run: raven provider set --api-key , then raven provider use /", + ) - if client == "codex": - return - if client == "azure": - if not p or not p.api_key or not p.api_base: - console.print("[red]Error: Azure OpenAI requires api_key and api_base.[/red]") - console.print("Set them in ~/.raven/config.json under providers.azure_openai section") - console.print("Use the model field to specify the deployment name.") - raise typer.Exit(1) - return - if not model.startswith("bedrock/") and not (p and p.api_key) and not (spec and (spec.is_oauth or spec.is_local)): - console.print("[red]Error: No API key configured.[/red]") - console.print("Set one in ~/.raven/config.json under providers section") - raise typer.Exit(1) + status = credential_status(provider_name, config.providers.get(provider_name), include_external=True) + if not status.ok: + raise MissingCredentialsError(status.summary, provider=provider_name) def make_provider(config: Config): @@ -121,24 +129,19 @@ def make_provider(config: Config): ) elif client == "azure": provider = AzureOpenAIProvider( - api_key=p.api_key, + api_key=p.effective_api_key, api_base=p.api_base, default_model=model, + deployment=getattr(p, "deployment", "") or "", + api_version=getattr(p, "api_version", "") or "2024-10-21", ) else: + from raven.providers.capabilities import wire_overrides from raven.providers.litellm_provider import LiteLLMProvider - # OpenRouter routes qwen3.x-27B through providers that default to - # reasoning mode (e.g. AtlasCloud): every chat completion emits - # ~800 chain-of-thought tokens and takes ~30s wall — fatal for - # interactive use and for high-volume benchmark runs. The - # ``reasoning.enabled=false`` flag is OpenRouter-specific and - # forwards through LiteLLM's ``extra_body``. - extra_body = None - if provider_name == "openrouter" and "qwen" in (model or "").lower(): - extra_body = {"reasoning": {"enabled": False}} + extra_body = wire_overrides(provider_name, model) or None provider = LiteLLMProvider( - api_key=p.api_key if p else None, + api_key=p.effective_api_key if p else None, api_base=config.get_api_base(model), default_model=model, extra_headers=p.extra_headers if p else None, diff --git a/raven/cli/commands.py b/raven/cli/commands.py index 0c453c29..472efd92 100644 --- a/raven/cli/commands.py +++ b/raven/cli/commands.py @@ -150,9 +150,20 @@ def main( def run() -> None: """Console-script entry point.""" from raven.config.loader import ConfigReadError + from raven.providers.auth import MissingCredentialsError try: app() + except MissingCredentialsError as exc: + # The gate is decided in `providers.auth` because three entry points ask + # it; printing and exiting is this one's idiom, so it happens here rather + # than there. Rendered once for every command, like ConfigReadError. + from raven.cli._helpers import console + + console.print(f"[red]Error: {exc.summary}.[/red]") + if exc.remedy: + console.print(exc.remedy) + raise SystemExit(1) from exc except ConfigReadError as exc: # A config-write command (channels/provider/deep-research/onboard) hit an # unparseable config. The write layer already refused (file untouched); diff --git a/raven/cli/onboard_commands.py b/raven/cli/onboard_commands.py index b00f94ac..28d21884 100644 --- a/raven/cli/onboard_commands.py +++ b/raven/cli/onboard_commands.py @@ -47,9 +47,8 @@ CRED_LOCAL, CRED_OAUTH, credential_kind, - needs_public_model_prefix, - public_model_prefix, ) +from raven.providers.wire import stored_model_id class _ThemedConsole(Console): @@ -890,11 +889,16 @@ def _verify_provider(provider: str, *, skip_test: bool = False) -> tuple[bool, s status = result.get("status", "unknown") # Some direct providers (openai / anthropic / deepseek / gemini) ship no # base URL and rely on the SDK's built-in endpoint, so there's nothing to - # hit for a GET /v1/models pre-check — the probe reports "not_configured" - # because api_base is empty. That's NOT a real auth failure: skip the pre- - # check (the test message sent later exercises real connectivity via + # hit for a GET /v1/models pre-check. That's NOT a real auth failure: skip + # the pre-check (the test message sent later exercises real connectivity via # litellm) instead of dumping the user into the failure submenu. - if status == "not_configured" and "api_base" in (result.get("error") or ""): + # + # `no_probe_endpoint` is the probe saying exactly this. It used to say + # `not_configured` with "api_base" in the text, which is why the old + # condition read that way -- and a rename this caller does not follow puts + # every one of those providers into the failure submenu on the first step + # of onboarding. + if status == "no_probe_endpoint" or (status == "not_configured" and "api_base" in (result.get("error") or "")): if skip_test: console.print( _t( @@ -971,45 +975,10 @@ def _format_model_for_provider(provider: str, spec: Any, model_id: str) -> str: Mistral alongside OpenAI produced "mistral-large-latest", which resolves to OpenAI and spends OpenAI's key. - Its section name is LiteLLM's own name for the vendor, which is exactly the - prefix LiteLLM routes on, so that is what goes in front. + The rule itself is ``providers.wire.stored_model_id``; deciding it here as + well is what made the wizard and the TUI write one model two ways. """ - - if not model_id: - return model_id - if spec is None: - # LiteLLM's own spelling, not the normalized one. A config section may be - # written either way and both resolve, but the wire prefix has to be the - # name LiteLLM routes on -- it hyphenates three vendors, and handing it - # "nano_gpt/..." is rejected with "LLM Provider NOT provided", so the - # provider configured fine and then could not be called. - from raven.providers.registry import litellm_spelling - - prefix = litellm_spelling(provider) - return model_id if model_id.startswith(f"{prefix}/") else f"{prefix}/{model_id}" - # These providers are named by a prefix their own client strips back off - # (``minimax_oauth`` before signing, ``_strip_model_prefix`` for codex), so the - # id can carry it -- and has to: written bare, "gpt-5.6-sol" is claimed by - # OpenAI's keywords and the request goes somewhere it does not exist. - # - # Azure is not in the list and does not reach this function either: an endpoint - # provider is locked in as ``is_custom`` and its model is persisted directly, - # so nothing here decides its spelling. It would need the opposite treatment - # anyway -- the id is used verbatim as a deployment name in a URL path. - if needs_public_model_prefix(spec): - public_prefix = public_model_prefix(spec) - if model_id.startswith(f"{public_prefix}/"): - return model_id - return f"{public_prefix}/{model_id.split('/')[-1]}" - prefix = getattr(spec, "model_prefix", "") or "" - if not prefix: - return model_id - if model_id.startswith(f"{prefix}/"): - return model_id - for skip in getattr(spec, "skip_prefixes", ()) or (): - if model_id.startswith(skip): - return model_id - return f"{prefix}/{model_id}" + return stored_model_id(provider, model_id) def _pick_model( @@ -1186,13 +1155,26 @@ def _write_provider_fields(provider: str, fields: dict[str, Any]) -> None: raise typer.Exit(1) -def _persist_default_model(model: Optional[str]) -> None: - """Patch ``agents.defaults.model`` if we picked one.""" +def _persist_default_model(model: Optional[str], provider: str) -> None: + """Patch ``agents.defaults.model`` and the pin that overrides it. + + Both, always. ``agents.defaults.provider`` wins over whatever a model id + names, so writing the model alone leaves the wizard's own choice routed to + whichever provider was pinned before -- with that provider's key. The rule + for what to pin is ``providers.pin``, the same one the picker and + ``raven provider use`` ask. + """ if not model: return + from raven.config.loader import load_config from raven.config.update import set_default_model + from raven.providers import pin - set_default_model(model) + try: + pinned = load_config().agents.defaults.provider or "" + except Exception: + pinned = "" + set_default_model(model, provider=pin.resolve(model, provider=provider, pinned=pinned)) # --------------------------------------------------------------------------- @@ -1426,7 +1408,7 @@ def _rewind() -> None: _roll_back_provider_fields(provider, spec, old_key=old_key, old_base=old_base) _rewind() continue - _persist_default_model(chosen_model) + _persist_default_model(chosen_model, provider) return {"provider": provider, "model": chosen_model} @@ -1631,7 +1613,7 @@ def _resolve_model_with_test( # Custom endpoints were previously trusted without a test message — the # highest-typo-risk case. Send the real probe (it builds from the stored # config, so a wrong base_url / model id fails here, not at first chat). - _persist_default_model(custom_model) + _persist_default_model(custom_model, provider) if skip_test: return custom_model while True: @@ -1654,7 +1636,7 @@ def _resolve_model_with_test( user_provided_model=user_model_flag, non_interactive=non_interactive, ) - _persist_default_model(chosen) + _persist_default_model(chosen, provider) if skip_test: return chosen result = _run_test_probe( @@ -1716,7 +1698,7 @@ def _configure_existing_provider_model(*, non_interactive: bool) -> bool: user_provided_model=None, non_interactive=False, ) - _persist_default_model(chosen) + _persist_default_model(chosen, provider) result = _run_test_probe( provider, non_interactive=False, @@ -1869,7 +1851,9 @@ def _manage_existing_providers(*, non_interactive: bool) -> None: # re-pick instead of leaving a model whose provider has no key. from raven.config.update import set_default_model - set_default_model("") + # The pin goes with it: left behind it would route the next model + # the user picks to the provider whose key was just removed. + set_default_model("", provider="auto") console.print( _t( f" [green]✓ Removed {_provider_label(target)}'s configuration.[/green]", @@ -2768,8 +2752,11 @@ def _resolve_model_provider(model: str) -> Optional[str]: except KeyError: pass # No usable prefix → could be a bare custom-endpoint model. + from raven.config.schema import ProviderConfig + from raven.providers.auth import credential_status + custom = (_load_raw_config().get("providers") or {}).get("custom") or {} - if custom.get("apiKey"): + if credential_status("custom", ProviderConfig.model_validate(custom)).ok: return "custom" # A bare id that still matches a known provider head (rare; e.g. a direct # provider's bare default before prefixing) — accept the head if known. diff --git a/raven/cli/provider_commands.py b/raven/cli/provider_commands.py index 76b1cda4..312a5755 100644 --- a/raven/cli/provider_commands.py +++ b/raven/cli/provider_commands.py @@ -270,9 +270,16 @@ def _parse_provider_flags(extra_args: list[str], provider_name: str) -> dict[str - ``--api-key abc`` -> ``{"api_key": "abc"}`` - ``--api-key=abc`` -> ``{"api_key": "abc"}`` - ``--api-base X`` -> ``{"api_base": "X"}`` (kebab -> snake) - - ``--vertex true`` -> ``{"vertex": True}`` (bool string coerced) - - ``--no-vertex`` -> ``{"vertex": False}`` (bool negative) - - ``--vertex`` alone -> ``{"vertex": True}`` (bool positive) + - ``-- true`` -> ``{"": "true"}`` (string; the schema coerces) + - ``--no-`` -> ``{"": False}`` (bool negative) + - ``--`` alone -> ``{"": True}`` (bool positive) + + Values come back as written; only the two valueless forms produce a bool + here, and the schema coerces the rest on validation. The bool forms are + named generically because no provider declares a bool field today -- the one + that did (Gemini's ``vertex``) described a mechanism that never existed and + was removed. They stay, matching ``_parse_channel_flags``, so a provider + gaining one needs no parser change. Unknown fields raise ``typer.BadParameter`` pointing at ``provider show``. """ @@ -333,6 +340,42 @@ def _normalize(flag: str) -> str: return out +def _load_section(name: str) -> dict | None: + """This provider's stored fields, or None when it has no section yet.""" + from raven.config.update_providers import get_provider_config + + try: + return get_provider_config(name, redact_secrets=False) + except KeyError: + return None + + +def _ineffective_because(provider: str, model: str) -> list[str]: + """Reasons this model will not actually be used, despite being written. + + A stale ``agents.defaults.provider`` used to be the other one, and the note + for it told the user to edit a field no command wrote. Both surfaces that + change a model now write it by the same rule, so the note would be advice + about a state neither of them produces. + """ + from raven.config.loader import load_config + + try: + config = load_config() + except Exception: + return [] + + notes: list[str] = [] + section = config.providers.get(provider) + deployment = getattr(section, "deployment", "") if section else "" + if deployment: + notes.append( + f"providers.{provider}.deployment is set to {deployment!r}, which decides the " + f"deployment regardless of the model id." + ) + return notes + + def _register_config_commands(app: typer.Typer) -> None: """Attach config subcommands to ``provider_app``.""" app.info.no_args_is_help = True @@ -413,7 +456,7 @@ def provider_set_cmd( raven provider set openrouter --api-key sk-or-v1-... raven provider set azure-openai --api-key X --api-base https://... - raven provider set gemini --api-key K --vertex true + raven provider set gemini --api-key-list k1,k2 """ if _help_requested(ctx.args): _print_schema_table(name) @@ -480,6 +523,14 @@ def provider_test_cmd( "oauth_token_missing": (f"Run: raven provider login {name.replace('_', '-')}"), "network_error": "Check network / firewall / VPN settings", } + if result["status"] == "no_probe_endpoint": + # Not a failure: this probe pings `/models`, and these vendors do not + # publish one at an address we hold. Saying "failed" here told seven + # correctly configured providers they were broken. + console.print(f"[yellow]?[/yellow] {name} not probed: {result['error']}") + console.print(" [dim]Credentials are set; run a turn to exercise them.[/dim]") + return + hint = hints.get(result["status"], "") console.print(f"[red]✗[/red] {name} failed: {result['status']}") if hint: @@ -488,6 +539,76 @@ def provider_test_cmd( console.print(f" [dim]Detail: {result['error']}[/dim]") raise typer.Exit(1) + @app.command("use") + def provider_use_cmd( + model: str = typer.Argument(..., help="Model id, e.g. anthropic/claude-sonnet-5"), + provider: str = typer.Option("", "--provider", "-p", help="Provider serving it, when the id does not say"), + ): + """Make this the model the agent runs on. + + Changing it used to mean re-running the whole wizard: the TUI picker and + onboarding could both switch models and the CLI could not, so a user on a + headless box had six setup steps to walk to change one field. + + The id is stored the way every other surface stores it -- naming its + provider -- so the three cannot disagree about what was chosen. + """ + from raven.config.loader import load_config + from raven.config.update import set_default_model + from raven.providers import pin + from raven.providers.auth import credential_status + from raven.providers.catalog import describe + from raven.providers.wire import stored_model_id + + try: + pinned = load_config().agents.defaults.provider or "" + except Exception: + pinned = "" + # One rule for both entry points: the picker writes this field, and the + # CLI used to tell the user to hand-edit it instead. + resolved = pin.resolve(model, provider=provider, pinned=pinned) + if resolved is None: + console.print(f"[red]✗[/red] cannot tell which provider serves {model!r}.") + console.print(f" [dim]Write it as /{model}, or pass --provider.[/dim]") + raise typer.Exit(1) + + name = provider or (resolved if resolved != pin.AUTO else "") + if not name: + from raven.providers.registry import split_model_id + + name = split_model_id(model)[0] + + stored = stored_model_id(name, model) if name else model + previous = set_default_model(stored, provider=resolved) + + row = describe(name, stored) + label = f"{row.label} ([dim]{stored}[/dim])" if row.described else stored + console.print(f"[green]✓[/green] default model: {label}") + if previous and previous != stored: + console.print(f" [dim]was {previous}[/dim]") + + # Reported rather than refused: choosing a model before configuring its + # provider is a normal order to do things in, and the startup gate says + # the same thing again if it is still missing then. + status = credential_status(name, _load_section(name), include_external=True) + if not status.ok: + console.print(f" [yellow]![/yellow] {status.summary}") + if resolved == pin.AUTO: + # "buy a key from this vendor" is the wrong advice for someone + # already paying a gateway to serve that vendor's models. Routing + # is left on auto precisely so the gateway can answer. + console.print( + f" [dim]Routing stays on auto, so a configured gateway can serve it. " + f"To pin the gateway instead, write it as /{stored}.[/dim]" + ) + + # An Azure deployment can still make this write have no effect, and it + # fails silently otherwise: the command reports success, the file + # changes, and requests keep going where they went before. The stale-pin + # case used to be the other one; this command now writes the pin itself. + for note in _ineffective_because(name, stored): + console.print(f" [yellow]![/yellow] {note}") + @app.command("reset") def provider_reset_cmd( name: str = typer.Argument(..., help="Provider name"), diff --git a/raven/cli/sentinel_commands.py b/raven/cli/sentinel_commands.py index e0c0efa0..3f8f8ed1 100644 --- a/raven/cli/sentinel_commands.py +++ b/raven/cli/sentinel_commands.py @@ -179,9 +179,6 @@ def sentinel_tick( base_config = ec_config.base ws = Path(workspace) if workspace else get_workspace_path() provider = make_provider(base_config) - # make_provider returns a tuple in some versions; normalise. - if isinstance(provider, tuple): - provider = provider[0] model = provider.get_default_model() # Frozen clock for eval. ``_kwargs`` is unpacked into every constructor @@ -1139,8 +1136,7 @@ def sentinel_behaviors_rebuild( "[yellow]behaviors_extract.enabled is False in config — " "rebuild will run but no future ticks will refresh.[/yellow]", ) - provider_pair = make_provider(cfg.base) - provider = provider_pair[0] if isinstance(provider_pair, tuple) else provider_pair + provider = make_provider(cfg.base) model = cfg.sentinel.behaviors_extract.model or cfg.sentinel.evaluator_model or provider.get_default_model() store = MemoryStore(ws) session_manager = SessionManager(ws) diff --git a/raven/cli/status_commands.py b/raven/cli/status_commands.py index a85b0705..6d3beba2 100644 --- a/raven/cli/status_commands.py +++ b/raven/cli/status_commands.py @@ -49,8 +49,14 @@ def status(): api_base = (section.api_base if section else None) or info["api_base"] state = f"[green]✓ {api_base}[/green]" if api_base else "[dim]not set[/dim]" else: - configured = bool(section and section.api_key) or info["configured"] - state = "[green]✓[/green]" if configured else "[dim]not set[/dim]" + # `providers.auth`, like every other gate. Reading `api_key` + # here made this the one place that called Azure configured + # with a key and no address, and missed a Gemini section + # holding only `api_key_list`. + from raven.providers.auth import credential_status + + status = credential_status(info["name"], section, include_external=True) + state = "[green]✓[/green]" if status.ok else "[dim]not set[/dim]" console.print(f"{label}: {state}") diff --git a/raven/cli/tui_commands.py b/raven/cli/tui_commands.py index caf78ee0..3c53367e 100644 --- a/raven/cli/tui_commands.py +++ b/raven/cli/tui_commands.py @@ -397,6 +397,7 @@ def _build_tui_agent_loop(): """ from pydantic import ValidationError + from raven.providers.auth import MissingCredentialsError from raven.tui_rpc.errors import InternalError try: @@ -479,6 +480,18 @@ def _build_tui_agent_loop(): # scheduler and its reply is fanned out as a cron.delivered event. return agent_loop + except MissingCredentialsError as e: + # Not a crash: the install simply is not finished. Surfaced as the + # sentence that says which provider needs what, where the generic + # handler below reported `exception_message: "1"` -- `typer.Exit` + # stringified -- and put the real one in a log file. + from loguru import logger as _logger + + _logger.warning("tui: provider not usable: {}", e.summary) + raise InternalError( + e.summary, + data={"reason": "missing_credentials", "provider": e.provider, "remedy": e.remedy}, + ) from e except (*_TUI_INIT_CRASH_TYPES, ValidationError) as e: from loguru import logger as _logger diff --git a/raven/config/schema.py b/raven/config/schema.py index b4cefb30..989f874e 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -314,6 +314,25 @@ class CronConfig(Base): """Default IANA timezone for cron expressions without explicit ``--tz``.""" +class ModelOverlay(Base): + """A name for a model no catalogue carries. + + A self-hosted deployment serves whatever was put there, and a model released + since the bundled snapshot is in no table yet, so the picker falls back to + showing the id. That is usually fine -- the id is the name the user gave + their own deployment -- but it leaves no way to label several of them. + + Only what a person states about presentation. Token accounting is not in + scope here -- `agents.defaults.contextWindowTokens` / `maxTokens` already + hold it. What has no knob at all is a *price* for an endpoint no catalogue + prices; such a deployment reports unknown spend rather than borrowing a + hosted model's rate. Adding one is a separate ask. + """ + + label: str = "" + description: str = "" + + class ProviderConfig(Base): """LLM provider configuration.""" @@ -321,40 +340,69 @@ class ProviderConfig(Base): api_base: str | None = None extra_headers: dict[str, str] | None = None # Custom headers (e.g. APP-Code for AiHubMix) models: list[str] = Field(default_factory=list) # User-curated model names for the picker + # Keyed by model id, in any spelling: what the user knows about a model that + # the catalogues do not. Deliberately additive rather than a change to + # `models` -- that list already lets a model be added, and what was missing + # was a way to describe one, so no config has to be rewritten to get it. + model_overlay: dict[str, ModelOverlay] = Field(default_factory=dict) + + @property + def effective_api_key(self) -> str: + """The key to send, which is not always the ``api_key`` field. + + Declared on the base so every call site can ask without knowing which + providers keep their key somewhere else. Gemini accepts a list, and a + section holding only that list handed LiteLLM an empty string: the + request left with no credential and failed at the API, having passed + every check that only asked whether credentials existed. + """ + return self.api_key + + +class AzureProviderConfig(ProviderConfig): + """Azure OpenAI, whose connection needs more than a key and an address. + + A deployment is a name the tenant gives one model, and it goes into the + request URL's path. It used to be read off ``agents.defaults.model``, which + made a model id double as a connection parameter: the id could carry no + prefix without the prefix landing in the path, so Azure was the one provider + whose ids had to be spelled differently from everyone else's. Declared here, + the model id is free to be a model id. + + ``api_version`` was hardcoded in the client, so a tenant on a different one + had no way to say so. + """ + + deployment: str = "" # falls back to the model id, for configs written before this field + api_version: str = "2024-10-21" class GeminiProviderConfig(ProviderConfig): - """Gemini provider configuration with Vertex AI and multi-key support. + """Gemini, which accepts several keys under one section. - Example YAML: + Example: gemini: - vertex: true - api_key_list: + apiKeyList: - "key1" - "key2" - """ - vertex: bool = False # When true, sets GOOGLE_GENAI_USE_VERTEXAI=True for Vertex AI - api_key_list: list[str] = Field(default_factory=list) # Multiple API keys for rotation - - def next_api_key(self) -> str: - """Return the next API key using round-robin rotation. - - Falls back to single api_key if api_key_list is empty. - """ - import itertools + A ``vertex`` flag used to sit here, documented as setting + ``GOOGLE_GENAI_USE_VERTEXAI``. Nothing read it, and it could not have worked: + that variable belongs to the google-genai SDK, while requests go through + LiteLLM, which does not read it and reaches Vertex as a separate provider + (``vertex_ai``) needing ``VERTEXAI_PROJECT`` and ``VERTEXAI_LOCATION``. It was + settable from the CLI and covered by tests, so it read as a supported feature + while doing nothing at all. Reaching Vertex is a change to how a request is + routed, not a boolean on a key. + """ - if not hasattr(self, "_key_cycle"): - keys = self.api_key_list if self.api_key_list else ([self.api_key] if self.api_key else []) - object.__setattr__(self, "_key_cycle", itertools.cycle(keys) if keys else None) - cycle = getattr(self, "_key_cycle", None) - if cycle is None: - return self.api_key or "" - return next(cycle) + #: Several keys may be listed; the first is used. Round-robin rotation was + #: declared here once and never called -- listing keys and silently using one + #: is the honest description of what happens. + api_key_list: list[str] = Field(default_factory=list) @property def effective_api_key(self) -> str: - """Get the current effective API key (first from list, or single key).""" if self.api_key_list: return self.api_key_list[0] return self.api_key @@ -380,7 +428,7 @@ def _prefer_set_values(base: dict[str, Any], winner: dict[str, Any]) -> dict[str return merged -def _has_credentials(config: "ProviderConfig", spec: Any) -> bool: +def _has_credentials(config: "ProviderConfig", spec: Any, name: str = "") -> bool: """Is this section actually usable, or just a placeholder? Every declared provider exists as an empty section whether or not the user @@ -388,12 +436,18 @@ def _has_credentials(config: "ProviderConfig", spec: Any) -> bool: stand in for evidence either: `is_local` used to answer with no api_base at all, and an empty declared section then beat the credentials the user had really written under one of that provider's other names. + + The rule itself lives in `providers.auth`, because deciding it here as well + is what made a Gemini section holding only `api_key_list` invisible to + routing while `provider list` showed it as configured. + + A vendor Raven carries no spec for reaches this too -- the passthrough route, + where the section name is all there is -- so the name is passed separately + rather than read off a spec that may not exist. """ - if spec.is_oauth: - return True # credentials live in a token file, not the config section - if spec.is_local: - return bool(config.api_base) - return bool(config.api_key) + from raven.providers.auth import credential_status + + return credential_status(name or (spec.name if spec else ""), config, spec=spec).ok class ProvidersConfig(Base): @@ -452,7 +506,7 @@ def _merge_renamed_sections(cls, data: Any) -> Any: return merged custom: ProviderConfig = Field(default_factory=ProviderConfig) # Any OpenAI-compatible endpoint - azure_openai: ProviderConfig = Field(default_factory=ProviderConfig) # Azure OpenAI (model = deployment name) + azure_openai: AzureProviderConfig = Field(default_factory=AzureProviderConfig) # Azure OpenAI anthropic: ProviderConfig = Field(default_factory=ProviderConfig) openai: ProviderConfig = Field(default_factory=ProviderConfig) openrouter: ProviderConfig = Field(default_factory=ProviderConfig) @@ -763,7 +817,13 @@ def effective_media_config(self) -> MediaGenConfig: def _match_provider(self, model: str | None = None) -> tuple["ProviderConfig | None", str | None]: """Match provider config and its registry name. Returns (config, spec_name).""" - from raven.providers.registry import PROVIDERS, canonical_provider_name, find_by_keywords, split_model_id + from raven.providers.registry import ( + PROVIDERS, + canonical_provider_name, + find_by_keywords, + find_by_name, + split_model_id, + ) forced = self.agents.defaults.provider if forced != "auto": @@ -789,9 +849,15 @@ def _match_provider(self, model: str | None = None) -> tuple["ProviderConfig | N # Explicit prefix naming a provider Raven has no spec for: LiteLLM knows # the vendor, so credentials under that name are enough to reach it. - if prefix: + # + # Only where there is genuinely no spec. A provider that has one has + # already been offered above and turned down for want of credentials -- + # letting it back in here on `api_key` alone reinstated exactly the + # material this rejected it for missing: Azure with a key and no address + # routed here, while display and startup both called it unconfigured. + if prefix and find_by_name(prefix) is None: passthrough = self.providers.get(prefix) - if passthrough and passthrough.api_key: + if passthrough and _has_credentials(passthrough, None, prefix): return passthrough, canonical_provider_name(prefix) # Fallback: configured local providers can route models without @@ -821,7 +887,7 @@ def _match_provider(self, model: str | None = None) -> tuple["ProviderConfig | N if spec.is_oauth: continue p = self.providers.get(spec.name) - if p and p.api_key: + if p and _has_credentials(p, spec): return p, spec.name return None, None @@ -838,7 +904,7 @@ def get_provider_name(self, model: str | None = None) -> str | None: def get_api_key(self, model: str | None = None) -> str | None: """Get API key for the given model. Falls back to first available key.""" p = self.get_provider(model) - return p.api_key if p else None + return p.effective_api_key if p else None def get_api_base(self, model: str | None = None) -> str | None: """Get API base URL for the given model. Applies default URLs for gateway/local providers.""" diff --git a/raven/config/update.py b/raven/config/update.py index 4b952789..9d0a2b33 100644 --- a/raven/config/update.py +++ b/raven/config/update.py @@ -185,6 +185,7 @@ def set_language( def set_default_model( model: str, *, + provider: str | None = None, config_path: Path | None = None, ) -> str | None: """Patch ``agents.defaults.model`` on the on-disk config. Returns previous value. @@ -193,14 +194,22 @@ def set_default_model( needs to swap the default model to one that matches the chosen provider (otherwise ``raven agent`` would still route to whatever the freshly created ``Config()`` baked in, which is typically a different vendor). + + ``provider`` writes ``agents.defaults.provider`` in the same patch. That field + overrides what a model id says, so leaving it behind lets a stale pin route + the new model to the old vendor -- with the old vendor's key -- while the + write that was just reported as successful changes nothing. Callers that do + not know which provider serves the model pass None and leave it alone. """ path = config_path or get_config_path() data = read_raw_or_raise(path) defaults = data.setdefault("agents", {}).setdefault("defaults", {}) prev = defaults.get("model") defaults["model"] = model + if provider is not None: + defaults["provider"] = provider _write_atomic(path, data) - logger.info("config/update: default model set to {} (was {})", model, prev) + logger.info("config/update: default model set to {} (was {}), provider={}", model, prev, provider) return prev diff --git a/raven/config/update_providers.py b/raven/config/update_providers.py index 95ba9ea1..5842e3ee 100644 --- a/raven/config/update_providers.py +++ b/raven/config/update_providers.py @@ -436,22 +436,9 @@ def _oauth_token_path(provider_name: str) -> Path: has a way to override its directory, so a second derivation is wrong exactly when a user has taken one. """ - from raven.config.paths import get_oauth_dir - - if provider_name == "github_copilot": - return _copilot_token_dir() / _COPILOT_TOKEN_FILES[0] - - if provider_name == "openai_codex": - from raven.providers.chatgpt_token import auth_file - - return auth_file() - - if provider_name in {"minimax_global", "minimax_cn"}: - from raven.providers.minimax_oauth import token_path - - return token_path("global" if provider_name == "minimax_global" else "cn") + from raven.providers.auth import credential_files - return get_oauth_dir() / f"{provider_name}.json" + return credential_files(provider_name)[0] def _copilot_token_dir() -> Path: @@ -506,11 +493,9 @@ def oauth_credential_files(provider_name: str) -> list[Path]: working credential -- and a sign-in has to restrict all of them, for the same reason in the other direction. """ - if provider_name == "github_copilot": - token_dir = _copilot_token_dir() - return [token_dir / filename for filename in _COPILOT_TOKEN_FILES] + from raven.providers.auth import credential_files - return [_oauth_token_path(provider_name)] + return credential_files(provider_name) # --------------------------------------------------------------------------- @@ -583,15 +568,17 @@ def list_providers(*, config_path: Path | None = None) -> list[dict[str, Any]]: api_base = getattr(instance, "api_base", None) api_key_list = list(getattr(instance, "api_key_list", []) or []) + # One rule for every gate: this used to accept a Gemini section holding + # only `api_key_list` that routing then skipped and startup refused. + from raven.providers.auth import credential_status + + configured = credential_status(fname, instance, spec=spec, include_external=True).ok if is_oauth: - configured = _oauth_credentials_present(fname) api_key_redacted = "OAuth token" if configured else "(empty)" elif is_local: - configured = bool(api_base) or bool(api_key) api_key_redacted = "(not needed for local)" if not api_key else "****set****" else: - configured = bool(api_key) or bool(api_key_list) - api_key_redacted = "****set****" if configured else "(empty)" + api_key_redacted = "****set****" if (api_key or api_key_list) else "(empty)" out.append( { @@ -701,6 +688,16 @@ def set_provider_fields( leaf_cls, leaf_field = _walk_nested_path(cls, path_key) leaf_info = leaf_cls.model_fields[leaf_field] coerced = _coerce_value(raw_val, leaf_info.annotation) + if path_key == "models" and isinstance(coerced, list): + # The third way a model id gets written down, and the one that used + # to skip the contract: `provider set --models x` stored a bare id + # while the picker and the wizard stored a qualified one. Identity + # still matched, so nothing broke -- which is exactly how the two + # spellings coexisted last time, until a delete silently matched + # neither. + from raven.providers.wire import stored_model_id + + coerced = [stored_model_id(name, str(m)) for m in coerced] prev[path_key] = _set_nested(path_key, coerced, working) validated = cls.model_validate(working) @@ -740,8 +737,8 @@ def reset_provider( Two cleanup paths run automatically, dispatched on ``ProviderSpec.is_oauth``: 1. **Config fields** — always rewritten to whatever a fresh Pydantic - instance produces (``api_key=""``, ``api_base=None``, ``vertex=False`` - for Gemini, ``api_key_list=[]`` etc.). For OAuth providers those are + instance produces (``api_key=""``, ``api_base=None``, + ``api_key_list=[]`` for Gemini, etc.). For OAuth providers those are already at defaults, so the write is a no-op for them but harmless. 2. **OAuth credential files** (``is_oauth=True``) — unlinked from disk so the @@ -800,11 +797,16 @@ def add_provider_model( Returns the new model list. Raises KeyError for an unknown provider. """ + from raven.providers.wire import merge_key + name = canonical_provider_name(name) path = config_path or get_config_path() data = read_raw_or_raise(path) cls, models = _load_provider_models(name, data) - if model not in models: + # By identity, not by string: the same model written two ways used to land + # in the list twice, and neither entry could then be removed by the other's + # spelling. + if merge_key(name, model) not in {merge_key(name, m) for m in models}: models.append(model) section = _raw_section(data, name) section["models"] = models @@ -824,12 +826,17 @@ def remove_provider_model( Returns the new model list. Raises KeyError for an unknown provider. """ + from raven.providers.wire import merge_key + name = canonical_provider_name(name) path = config_path or get_config_path() data = read_raw_or_raise(path) cls, models = _load_provider_models(name, data) - if model in models: - models = [m for m in models if m != model] + # Whatever spelling the caller holds removes every spelling of that model: + # the write paths used to disagree, so a list could hold one model twice. + target = merge_key(name, model) + if target in {merge_key(name, m) for m in models}: + models = [m for m in models if merge_key(name, m) != target] section = _raw_section(data, name) section["models"] = models validated = cls.model_validate(section) @@ -901,6 +908,7 @@ def test_provider( api_key = cfg.get("api_key") or "" api_base = cfg.get("api_base") or (spec.default_api_base if spec else "") or "" + derived_api_base = False # Before the token fetch below, which asks a question this backend does not # answer: its catalogue is the credential check. @@ -966,14 +974,37 @@ def test_provider( } if not api_base: + # Asked here and not above: the branches in between return for the + # families whose credential check is not an HTTP ping, and one of them + # is Copilot -- whose driver starts a GitHub device flow when LiteLLM is + # asked to resolve it. Deriving eagerly put that flow before the branch + # that avoids it and hung `provider test github-copilot` on that login. + # Derived rather than declared, and tracked as such: a 404 from an + # address we guessed says the vendor has no models route there, while a + # 404 from one the user typed is a typo they need to see. + api_base = _litellm_api_base(spec) + derived_api_base = bool(api_base) + + if not api_base: + # No address, and for most of these there is nothing the user could have + # supplied: the endpoint is compiled into the vendor's SDK, so there is + # no `/models` to ping. Reporting `not_configured` told seven correctly + # configured providers they were not set up, and pointed at a key they + # had already set. Say what is true instead -- the credential is present + # and this probe cannot reach the vendor. + needs_user_address = bool(spec and (spec.is_local or spec.name == "azure_openai")) return { "ok": False, - "status": "not_configured", + "status": "not_configured" if needs_user_address else "no_probe_endpoint", "elapsed_ms": 0, "http_status": None, "models_count": None, "model_ids": None, - "error": "api_base is empty and provider has no default", + "error": ( + "api_base is empty and provider has no default" + if needs_user_address + else "credential present; this vendor publishes no models endpoint to ping" + ), } url = api_base.rstrip("/") + "/models" @@ -984,7 +1015,57 @@ def test_provider( if spec and spec.name in {"minimax_global", "minimax_cn"} and api_key: headers["x-api-key"] = api_key - return _probe_models_endpoint(url, headers, timeout_s=timeout_s, transport=transport) + result = _probe_models_endpoint(url, headers, timeout_s=timeout_s, transport=transport) + if derived_api_base and result.get("status") == "http_404": + # The address LiteLLM sends completions to is not always where the + # catalogue lives -- DeepSeek's is `/beta`, which has no `/models`. A 404 + # never says anything about the credential, so reporting a failure here + # would be the same lie in a new spelling. + return { + **result, + "ok": False, + "status": "no_probe_endpoint", + "error": "credential present; this vendor publishes no models endpoint to ping", + } + return result + + +def _litellm_api_base(spec: Any) -> str: + """The endpoint LiteLLM would send this vendor's request to, or "". + + Asked rather than tabulated: LiteLLM already knows, because it is the thing + that does the sending, and a second copy of these addresses is a second thing + to keep current. It answers for four of the ten providers that ship no + default; the rest compile the address into the vendor SDK and there is + nothing to return. + """ + if spec is None: + return "" + if spec.is_oauth: + # Asked before the id is built, because building it can hide the answer: + # `wire_model` strips the provider name entirely for the codex and azure + # shapes, so the guard below would be handed a bare "probe-model" and see + # nothing to object to. An OAuth provider's credential check is its own + # flow, never a models ping, so there is nothing here for it either way. + return "" + + from raven.providers.rates import _may_prompt + from raven.providers.wire import stored_model_id, wire_model + + # Asked of the stored form, not the wire form, for the same reason. + stored = stored_model_id(spec.name, "probe-model") + if _may_prompt(stored): + # Resolving one of these resolves its credentials on the way, and with no + # token file that prints a device code and blocks. One answer to "can this + # be handed to LiteLLM" for every caller -- see providers.rates. + return "" + try: + from raven.providers.litellm_setup import import_litellm + + _, _, _, base = import_litellm().get_llm_provider(model=wire_model(stored, spec=spec)) + except Exception: + return "" + return base or "" def _probe_models_endpoint( diff --git a/raven/providers/auth.py b/raven/providers/auth.py new file mode 100644 index 00000000..6132c223 --- /dev/null +++ b/raven/providers/auth.py @@ -0,0 +1,275 @@ +"""How a provider is connected to: what material it needs, and whether it is there. + +Authentication used to be described by one boolean (``ProviderSpec.is_oauth``) +and one string (``env_key``). Underneath sit shapes those two cannot express: +Azure needs a key *and* an address; Gemini takes a key *or* a list of them; +Bedrock needs neither because the environment already holds AWS credentials; +four providers hold a token in a file, written by three unrelated flows. + +Because the shape was not stated, "is this provider usable" was answered +independently wherever it was needed, and the answers diverged. Routing skipped +a Gemini section configured with ``api_key_list``; ``provider list`` showed the +same section as ready; startup refused to run on it. Azure with a key and no +address was accepted by routing and display and rejected at startup. + +So the requirement is declared here, once, as **an AND of OR-groups**: every +group must be satisfied, and any member satisfies its group. That is the whole +grammar, and it is enough for all eight shapes -- "key and address" is two +groups, "key or key list" is one group with two members. + +Callers ask :func:`credential_status`. What they must not do is re-derive the +answer from spec flags, which is what produced the divergence. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from raven.providers.registry import ProviderSpec + +#: A credential arrives as a token file rather than a config field. +KIND_DEVICE_FLOW = "device_flow" +#: The environment already holds it (AWS credential chain, Google ADC). +KIND_AMBIENT = "ambient" +#: An address is all that is needed; there is no credential. +KIND_NONE = "none" +#: A key held in the config. Which fields count is the method's ``requires``. +KIND_API_KEY = "api_key" + + +@dataclass(frozen=True) +class Requirement: + """One thing that must be present, satisfied by any of ``fields``. + + ``fields`` names config fields; an empty tuple means the material is not in + the config at all and ``token_file`` or ``ambient`` decides instead. + """ + + fields: tuple[str, ...] + label: str + hint: str = "" + + def satisfied_by(self, section: Any) -> bool: + return any(_present(section, name) for name in self.fields) + + +@dataclass(frozen=True) +class AuthMethod: + """One way of connecting to a provider. A provider may offer several.""" + + kind: str + requires: tuple[Requirement, ...] = () + #: Checked instead of ``requires`` when the credential is not in the config. + checks_token_file: bool = False + label: str = "" + + def missing(self, section: Any, provider: str, *, include_external: bool) -> list[Requirement]: + """What this method still needs. + + ``include_external`` decides whether material held outside the config is + looked at. Routing asks without it: it is choosing which section serves a + model id, it runs on every call, and an OAuth provider's section is + legitimately empty -- a token file read there would put disk I/O on the + hot path and make the choice depend on a sign-in that startup is the + right place to require. Display and startup ask with it, because both + report on what is true right now. + """ + if self.checks_token_file: + if not include_external: + return [] + return [] if _token_present(provider) else [_SIGN_IN(provider)] + return [req for req in self.requires if not req.satisfied_by(section)] + + +class MissingCredentialsError(Exception): + """A provider cannot be used because its credentials are absent. + + Raised where the gate is decided, not where it is reported. The check runs + behind three entry points -- the CLI, the gateway, and the TUI -- and used to + end in ``console.print`` plus ``typer.Exit``, which is one of them speaking. + Through the other two the message went to a log nobody was reading and the + user got ``internal_error`` with ``exception_message: "1"``: the exit code, + stringified. + """ + + def __init__(self, summary: str, *, provider: str = "", remedy: str = ""): + super().__init__(summary) + self.summary = summary + self.provider = provider + #: The command that fixes it, when there is one to name. + self.remedy = remedy + + +@dataclass(frozen=True) +class CredentialStatus: + """The single answer to "can this provider be used right now".""" + + provider: str + ok: bool + kind: str + missing: tuple[Requirement, ...] = () + #: Which declared method is satisfied, or the first one when none is. + method: AuthMethod | None = field(default=None, compare=False) + + @property + def summary(self) -> str: + """What to tell the user, naming the field rather than "No API key".""" + if self.ok: + return f"{self.provider} is configured" + if not self.missing: + return f"{self.provider} is not configured" + parts = [m.hint or m.label for m in self.missing] + return f"{self.provider} needs {', '.join(parts)}" + + +def _present(section: Any, name: str) -> bool: + """Is this field set to something usable, on a model or a plain dict? + + Sections reach here as both: the schema object on the routing path, a raw + mapping on the display path. + """ + if section is None: + return False + value = section.get(name) if isinstance(section, dict) else getattr(section, name, None) + if isinstance(value, (list, tuple)): + return any(bool(v) for v in value) + return bool(value) + + +def _token_present(provider: str) -> bool: + from raven.config.update_providers import _oauth_credentials_present + + return _oauth_credentials_present(provider) + + +def credential_files(provider: str) -> list[Path]: + """Every file a sign-in for this provider can leave behind. + + Asked of the module that writes each family rather than derived a second + time: each has its own override for where it puts things, so a second + derivation is wrong exactly when a user has taken one -- that is how + ``openai_codex`` came to be written under one name and read under another. + + Returns a list because a sign-in is not always one file: Copilot exchanges + its token for an API key with a longer life, and clearing the token alone + leaves a working credential behind. The first entry is the one that stands + for the credential when a single path is needed. + """ + from raven.config.paths import get_oauth_dir + + if provider == "github_copilot": + from raven.config.update_providers import _COPILOT_TOKEN_FILES, _copilot_token_dir + + return [_copilot_token_dir() / name for name in _COPILOT_TOKEN_FILES] + + if provider == "openai_codex": + from raven.providers.chatgpt_token import auth_file + + return [auth_file()] + + if provider in {"minimax_global", "minimax_cn"}: + from raven.providers.minimax_oauth import token_path + + return [token_path("global" if provider == "minimax_global" else "cn")] + + return [get_oauth_dir() / f"{provider}.json"] + + +def _SIGN_IN(provider: str) -> Requirement: # noqa: N802 - a constructor, named for the constant it stands in for + public = provider.replace("_", "-") + return Requirement((), "a sign-in", f"a sign-in -- run `raven provider login {public}`") + + +_KEY = Requirement(("api_key",), "an API key", "an API key -- run `raven provider set {public} --api-key `") +_KEY_OR_LIST = Requirement( + ("api_key", "api_key_list"), + "an API key", + "an API key -- run `raven provider set {public} --api-key ` (or --api-key-list k1,k2)", +) +_ADDRESS = Requirement(("api_base",), "an address", "an address -- run `raven provider set {public} --api-base `") + + +#: Declarations for the providers whose shape the spec flags cannot express. +#: Everyone else is derived by :func:`auth_methods` from the flags themselves, +#: so adding an ordinary key-based vendor still needs no entry here. +_DECLARED: dict[str, tuple[AuthMethod, ...]] = { + # A key or a list of them. The plural field is the one that shipped + # unreadable to the router while displaying as configured. + "gemini": (AuthMethod(KIND_API_KEY, (_KEY_OR_LIST,), label="API key"),), + # No entry for Azure or the generic endpoint: `requires_api_base` already + # derives exactly this, and a declaration identical to what the flags produce + # is a second statement of one fact -- the thing this module exists to end. + # The AWS credential chain already holds these; asking for a key would be + # asking for something the user does not have in this form. + "bedrock": (AuthMethod(KIND_AMBIENT, (), label="AWS credentials"),), +} + + +def auth_methods(spec: "ProviderSpec | None", name: str = "") -> tuple[AuthMethod, ...]: + """Every way this provider can be connected to, most preferred first.""" + provider = name or (spec.name if spec else "") + declared = _DECLARED.get(provider) + if declared: + return declared + if spec is None: + # A vendor Raven carries no spec for is reached with a key, like most. + return (AuthMethod(KIND_API_KEY, (_KEY,), label="API key"),) + if spec.is_oauth: + return (AuthMethod(KIND_DEVICE_FLOW, checks_token_file=True, label="sign-in"),) + if spec.is_local: + return (AuthMethod(KIND_NONE, (_ADDRESS,), label="address"),) + if spec.requires_api_base: + return (AuthMethod(KIND_API_KEY, (_KEY, _ADDRESS), label="key and endpoint"),) + return (AuthMethod(KIND_API_KEY, (_KEY,), label="API key"),) + + +def credential_status( + name: str, + section: Any, + *, + spec: "ProviderSpec | None" = None, + include_external: bool = False, +) -> CredentialStatus: + """Can this provider be used, and if not, what exactly is missing? + + ``section`` is the provider's config -- the schema object or the raw mapping, + either way. A provider offering several methods is usable when any one of + them is satisfied, which is what lets a vendor be reached by an ambient + credential when no key is set. + + ``include_external`` extends the question to material held outside the + config, which today means an OAuth token file. See ``AuthMethod.missing`` + for why routing deliberately asks without it. + """ + from raven.providers.registry import canonical_provider_name, find_by_name + + name = canonical_provider_name(name) + spec = spec if spec is not None else find_by_name(name) + methods = auth_methods(spec, name) + + unsatisfied: list[tuple[AuthMethod, tuple[Requirement, ...]]] = [] + for method in methods: + gap = method.missing(section, name, include_external=include_external) + if not gap: + return CredentialStatus(name, True, method.kind, (), method) + unsatisfied.append((method, tuple(_localize(req, name) for req in gap))) + + # Report against the first declared method: it is the preferred one, so its + # gap is the shortest path to a working provider. + method, gap = unsatisfied[0] + return CredentialStatus(name, False, method.kind, gap, method) + + +def _localize(req: Requirement, name: str) -> Requirement: + """Put the provider's own name into the hint, so it can be pasted. + + The hint names a command, not a config path: telling someone to hand-edit + `~/.raven/config.json` asks them to know a file layout the CLI exists to + hide, and the OAuth hint already gave a command. + """ + if "{public}" not in req.hint: + return req + return Requirement(req.fields, req.label, req.hint.replace("{public}", name.replace("_", "-"))) diff --git a/raven/providers/azure_openai_provider.py b/raven/providers/azure_openai_provider.py index 9a2e6e66..73a7d307 100644 --- a/raven/providers/azure_openai_provider.py +++ b/raven/providers/azure_openai_provider.py @@ -32,10 +32,15 @@ def __init__( api_key: str = "", api_base: str = "", default_model: str = "gpt-5.2-chat", + deployment: str = "", + api_version: str = "2024-10-21", ): super().__init__(api_key, api_base) self.default_model = default_model - self.api_version = "2024-10-21" + # Empty means "the model id names the deployment", which is how every + # config written before the field existed says it. + self.deployment = deployment + self.api_version = api_version # Validate required parameters if not api_key: @@ -49,9 +54,20 @@ def __init__( self.api_base = api_base def _build_chat_url(self, deployment_name: str) -> str: - """Build the Azure OpenAI chat completions URL.""" + """Build the Azure OpenAI chat completions URL. + + A configured ``deployment`` decides; otherwise the model id names it, as + it did before the field existed. Falling back rather than requiring the + field keeps working configs working -- and it is why the id may still not + carry a prefix in that case: whatever is here goes into the URL path. + """ # Azure OpenAI URL format: # https://{resource}.openai.azure.com/openai/deployments/{deployment}/chat/completions?api-version={version} + from raven.providers.registry import find_by_name + from raven.providers.wire import wire_model + + deployment_name = self.deployment or wire_model(deployment_name, spec=find_by_name("azure_openai")) + base_url = self.api_base if not base_url.endswith("/"): base_url += "/" diff --git a/raven/providers/base.py b/raven/providers/base.py index 19f42db4..1d628f6c 100644 --- a/raven/providers/base.py +++ b/raven/providers/base.py @@ -5,7 +5,7 @@ import random from abc import ABC, abstractmethod from collections.abc import AsyncIterator -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any from loguru import logger @@ -60,6 +60,12 @@ class ErrorClassification: should_fallback: bool = False should_compress: bool = False should_drop_tool_images: bool = False + #: The upstream refused the prompt-cache breakpoints specifically. Decided + #: here for the same reason the rest of this verdict is: a provider that + #: swallows the exception into a string loses the response body with it, and + #: whether ``str()`` carried that body is a property of the client that + #: raised it. Deciding while the exception is alive makes it one answer. + refuses_prompt_cache: bool = False @dataclass @@ -337,9 +343,24 @@ def classify_error( Precise when given the live exception (status code + class names); degrades to substring matching when the provider already swallowed it - into ``content``. Order matters: context-overflow and rate-limit are - checked before the generic 400/server buckets. + into ``content`` -- which is why every verdict, including + ``refuses_prompt_cache``, is decided here rather than downstream. """ + from raven.providers import prompt_cache + + verdict = cls._classify(exc, content) + if prompt_cache.is_rejection(exc if exc is not None else (content or "")): + return replace(verdict, refuses_prompt_cache=True) + return verdict + + @classmethod + def _classify( + cls, + exc: BaseException | None = None, + content: str | None = None, + ) -> ErrorClassification: + """The bucket this failure falls in. Order matters: context-overflow and + rate-limit are checked before the generic 400/server buckets.""" status = cls._extract_status_code(exc) names = cls._error_type_names(exc) msg = (content if content is not None else str(exc) if exc is not None else "").lower() @@ -496,8 +517,11 @@ async def _chat_attempt_with_retry( always carries an ``error_classification`` so the caller (model-chain fallback) can decide without re-classifying. """ + from raven.providers import prompt_cache + total_attempts = len(self._CHAT_RETRY_DELAYS) + 1 last_response: LLMResponse | None = None + dropped_cache_control = False for attempt in range(1, total_attempts + 1): exc: Exception | None = None try: @@ -525,6 +549,19 @@ async def _chat_attempt_with_retry( response.error_classification = classification last_response = response + # Why an upstream can refuse this at all: see + # ``providers.prompt_cache.suppress``. Learned from the refusal, once + # per model. The marks already in the payload were placed upstream by + # a token strategy, so they are taken off here; suppressing stops the + # provider adding its own back on the way out. + # Read off the verdict rather than re-derived here: by this point a + # provider may have turned the exception into a string. + if not dropped_cache_control and attempt < total_attempts and classification.refuses_prompt_cache: + dropped_cache_control = True + prompt_cache.suppress(model or getattr(self, "default_model", "") or "") + messages, tools = prompt_cache.strip(messages, tools) + continue + if not classification.retryable or attempt == total_attempts: return response @@ -574,9 +611,17 @@ async def chat_with_retry( if reasoning_effort is self._SENTINEL: reasoning_effort = self.generation.reasoning_effort + from raven.providers import prompt_cache + model_chain = [model, *(fallback_models or [])] response: LLMResponse | None = None for idx, current_model in enumerate(model_chain): + # The breakpoints in this payload were placed for whoever was asked + # first. A fallback is a different model, often a different vendor, + # and the field it does not read is billed rather than refused -- + # sending Anthropic's markers on to Gemini is what doubled a prompt. + if idx and not prompt_cache.accepts_cache_control(current_model or ""): + messages, tools = prompt_cache.strip(messages, tools) response = await self._chat_attempt_with_retry( messages=messages, tools=tools, diff --git a/raven/providers/capabilities.py b/raven/providers/capabilities.py index b8974fe1..0c328135 100644 --- a/raven/providers/capabilities.py +++ b/raven/providers/capabilities.py @@ -176,9 +176,9 @@ def vision_verdict( if _model_id_is_caller_chosen(model, provider, spec): return None - # Imported inside the call: pricing reaches back into this package, so a - # module-level import here would close the loop. - from raven.token_wise.pricing import openrouter_input_modalities, warm_catalog_in_background + # Imported inside the call: rates reaches back into this package's + # registry, so a module-level import here would close the loop. + from raven.providers.rates import openrouter_input_modalities, warm_catalog_in_background try: mods = openrouter_input_modalities(model) @@ -205,7 +205,7 @@ def supports_vision( telling the model to read it another way. Answered from the gateway catalog Raven already fetches and caches for - pricing (:func:`raven.token_wise.pricing.openrouter_input_modalities`), which + pricing (:func:`raven.providers.rates.openrouter_input_modalities`), which publishes ``input_modalities`` for every model it lists. That completeness is the reason it is the source rather than LiteLLM's price table: the table states ``supports_vision`` on under a third of its rows, and reading the @@ -283,3 +283,41 @@ def image_placeholder_text( "this endpoint cannot carry images in a tool result]" ) return body.strip() + + +#: Request-body extras a provider needs for particular models, declared rather +#: than branched on at the point a provider is built. +#: +#: Each entry is (provider, substring of the model id, body). A substring rather +#: than an id because a vendor's quirk covers a family; this one is deliberately +#: broad -- every qwen model behind OpenRouter, which is what the branch it +#: replaces matched too. +_WIRE_OVERRIDES: tuple[tuple[str, str, dict[str, Any]], ...] = ( + # OpenRouter routes qwen3.x through hosts that default to reasoning mode + # (AtlasCloud among them): every completion emits ~800 chain-of-thought + # tokens and takes ~30s wall, which is fatal interactively and for volume + # benchmark runs. The flag is OpenRouter's own and rides in extra_body. + ("openrouter", "qwen", {"reasoning": {"enabled": False}}), +) + + +def wire_overrides(provider: str | None, model: str | None) -> dict[str, Any]: + """Extras to send in the request body for this provider and model. + + Lived as an ``if provider_name == ... and ... in model`` inside the factory, + because a fact about one model family behind one gateway had nowhere else to + go. Declared here it sits with the other per-model facts, and a second one + does not mean a second branch in provider construction. + """ + from raven.providers.registry import normalize_provider_name + + if not provider or not model: + return {} + + name = normalize_provider_name(provider) + lowered = model.lower() + out: dict[str, Any] = {} + for owner, needle, body in _WIRE_OVERRIDES: + if normalize_provider_name(owner) == name and needle in lowered: + out.update(body) + return out diff --git a/raven/providers/catalog.py b/raven/providers/catalog.py new file mode 100644 index 00000000..a6e4b30f --- /dev/null +++ b/raven/providers/catalog.py @@ -0,0 +1,179 @@ +"""What a model is called and what it is for, for the surfaces people read. + +A picker showing `anthropic/claude-sonnet-4-6` is showing an identifier. What a +person choosing a model wants is its name, roughly what it is good at, how much +context it takes and how recent it is -- none of which LiteLLM's table carries, +because that table exists to price and route. + +So there are two catalogue sources and they answer different questions: + +* LiteLLM's own table decides prices and limits used in a request. It ships with + the dependency and needs no network. It also carries capability flags, which + Raven deliberately does not read: its `supports_prompt_caching` asks whether a + model caches at all, while what a request needs to know is whether the provider + accepts `cache_control` blocks -- `ProviderSpec`'s field of the same name. +* the models.dev snapshot decides labels, and prices a finished call. It carries + a name, a one-line description, and the vendor's published cost per model. + Context windows and capability flags are deliberately absent: those shape the + *next* request -- a window sizes trimming, a flag picks a wire shape -- and a + second source for them is a second answer. + +Keeping the split is the point rather than an implementation detail. The snapshot +is community-maintained data; if it goes stale, wrong, or missing, the cost is a +model shown by its id instead of its name, or a total that is off. It can never +cause a wrong request, because nothing that shapes one reads it. + +The snapshot ships with Raven so a fresh install labels models offline and tests +never reach the network. Regenerate with +``scripts/refresh_models_dev_snapshot.py``. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, replace +from functools import lru_cache +from pathlib import Path +from typing import TYPE_CHECKING + +from loguru import logger + +if TYPE_CHECKING: + from raven.config.schema import ModelOverlay + +SNAPSHOT = Path(__file__).parent / "data" / "models_dev.json" + +#: Where a row's facts came from, kept on the row so a surface can tell a +#: label it can trust from an id it is falling back to. +SOURCE_SNAPSHOT = "snapshot" +SOURCE_ID_ONLY = "id-only" +#: The user described it themselves, which beats any catalogue. +SOURCE_OVERLAY = "overlay" + + +@dataclass(frozen=True) +class ModelRow: + """One model, as a person reads it.""" + + ref: str + provider: str + label: str + source: str + description: str = "" + + @property + def described(self) -> bool: + return self.source != SOURCE_ID_ONLY + + +@lru_cache(maxsize=1) +def _snapshot() -> dict[str, dict]: + """The bundled labels, or nothing when they cannot be read. + + Never raises: a missing or corrupt snapshot must cost labels, not startup. + """ + try: + return json.loads(SNAPSHOT.read_text(encoding="utf-8")) + except Exception as exc: # pragma: no cover - only on a damaged install + logger.debug(f"model label snapshot unavailable: {exc}") + return {} + + +def describe(provider: str, model: str, *, overlay: "ModelOverlay | None" = None) -> ModelRow: + """Everything known about this model for display purposes. + + Falls back to the id as its own label, so a caller can render the result + unconditionally: a model the snapshot has never heard of -- one released + since the last refresh, or served by a local deployment -- still comes back + as a row rather than as nothing to show. + """ + from raven.providers.registry import canonical_provider_name + from raven.providers.wire import split_model_id, stored_model_id + + provider = canonical_provider_name(provider) + ref = stored_model_id(provider, model) + entry = _snapshot().get(provider, {}).get("models", {}).get(_vendor_id(provider, model)) + + if entry: + row = ModelRow( + ref=ref, + provider=provider, + label=entry.get("name") or ref, + source=SOURCE_SNAPSHOT, + description=entry.get("description") or "", + ) + else: + row = ModelRow(ref=ref, provider=provider, label=split_model_id(ref)[1] or ref, source=SOURCE_ID_ONLY) + + return _with_overlay(row, overlay) + + +def model_cost(model: str) -> dict | None: + """The vendor's own published rates for this model, or None. + + Keyed by provider, which is the point: reading a price out of a flat + cross-vendor table answers a self-hosted deployment with a hosted vendor's + figure. ``model`` is a stored id, so the provider it names is the one asked. + + Prices are the one runtime number this file carries, and only because they + are reported after a call rather than used to shape one -- see the module + docstring for where that line is drawn. + """ + from raven.providers.registry import canonical_provider_name, find_by_model, split_model_id + + # The id has to name its provider. `find_by_model` falls back to keyword + # matching for a bare id, which reads across vendors: "qwen3-32b" matched + # DashScope and was priced at DashScope's rate whoever was actually serving + # it. That is the same borrowing the openrouter tier was gated to stop, one + # tier down. + # + # A bare id left by an older version therefore prices as unknown. That is not + # worth a compatibility path: both surfaces that write a model now store it + # qualified, so picking the model once restores the figure. + if not split_model_id(model)[0]: + return None + spec = find_by_model(model) + provider = spec.name if spec else split_model_id(model)[0] + if not provider: + return None + entry = _snapshot().get(canonical_provider_name(provider), {}).get("models", {}).get(_vendor_id(provider, model)) + cost = entry.get("cost") if isinstance(entry, dict) else None + return cost if isinstance(cost, dict) else None + + +def _with_overlay(row: ModelRow, overlay: "ModelOverlay | None") -> ModelRow: + """Let what the user stated beat what a catalogue guessed. + + The user is describing their own deployment, so they are the authority on + it -- and for a model no catalogue carries, they are the only one. Fields + left unset in the overlay keep the catalogue's answer rather than blanking + it, so stating one fact does not erase the rest. + """ + if overlay is None: + return row + + changed = { + "label": overlay.label or row.label, + "description": overlay.description or row.description, + } + described = row.described or bool(overlay.label or overlay.description) + return replace(row, **changed, source=SOURCE_OVERLAY if described else row.source) + + +def _vendor_id(provider: str, model: str) -> str: + """The vendor's own id, which is how the snapshot is keyed. + + A stored id names its provider and the snapshot does not repeat that, so the + prefix comes off before the lookup -- including a gateway's, whose rows are + filed under the upstream vendor's id. + """ + from raven.providers.registry import find_by_name + from raven.providers.wire import split_model_id + + spec = find_by_name(provider) + head, rest = split_model_id(model or "") + if head and spec and head in spec.route_names: + return rest + if head and head == provider: + return rest + return model or "" diff --git a/raven/providers/common_models.py b/raven/providers/common_models.py index 9fc08f23..d5da8c65 100644 --- a/raven/providers/common_models.py +++ b/raven/providers/common_models.py @@ -204,7 +204,7 @@ def litellm_models_for(slug: str) -> list[str]: do not -- and offering a bare id would route it by keyword rather than to the provider the user picked. """ - from raven.providers.registry import find_by_name, litellm_spelling, normalize_provider_name + from raven.providers.registry import find_by_name, litellm_spelling spec = find_by_name(slug) if spec is None: @@ -228,23 +228,19 @@ def litellm_models_for(slug: str) -> list[str]: bare = model[len(prefix) + 1 :] if model.startswith(f"{prefix}/") else model out.append(f"{prefix}/{bare}") return out - if normalize_provider_name(spec.model_prefix) not in spec.route_names: - # The wire prefix names somebody else -- this provider is reached through - # another vendor's driver. Prefixing candidates with it would hand the - # user ids that resolve to the driver's owner instead of to the provider - # they picked. Today the catalogue has no rows for those five, so the - # loop below would be empty anyway; the guard states the rule rather than - # relying on what LiteLLM happens to contain. - return [] + from raven.providers.wire import merge_key, stored_model_id + index = _litellm_chat_models_by_provider() - prefix = spec.model_prefix out: list[str] = [] seen: set[str] = set() for route_name in sorted(spec.route_names): for model in index.get(route_name, ()): bare = model.split("/", 1)[1] if "/" in model else model - full = f"{prefix}/{bare}" if prefix else bare - if full not in seen: - seen.add(full) + # One spelling for a candidate and for a stored pick, so choosing an + # offered model cannot write a second entry for one already listed. + full = stored_model_id(spec.name, bare) + key = merge_key(spec.name, full) + if key not in seen: + seen.add(key) out.append(full) return out diff --git a/raven/providers/data/models_dev.json b/raven/providers/data/models_dev.json new file mode 100644 index 00000000..863390b1 --- /dev/null +++ b/raven/providers/data/models_dev.json @@ -0,0 +1 @@ +{"_source":{"ref":"dev","repo":"anomalyco/models.dev","sha":"beca303ea34379c9c863e9229fd01681d7bdd45e"},"aihubmix":{"models":{"alicloud-deepseek-v4-flash":{"cost":{"cache_read":0.028,"input":0.14,"output":0.28},"description":"Fast DeepSeek model for efficient chat, coding help, and agent loops","name":"DeepSeek V4 Flash (Alibaba Cloud)"},"alicloud-deepseek-v4-pro":{"cost":{"cache_read":0.13,"input":1.69,"output":3.38},"description":"Flagship DeepSeek model for coding, reasoning, and agentic work","name":"DeepSeek V4 Pro (Alibaba Cloud)"},"alicloud-glm-5.1":{"cost":{"cache_read":0.169,"cache_write":1.05625,"input":0.84,"output":3.38},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM-5.1 (Alibaba Cloud)"},"claude-fable-5":{"cost":{"cache_read":1.1,"cache_write":13.75,"input":11,"output":55},"description":"Claude model for creative writing, analysis, and controlled agent workflows","name":"Claude Fable 5"},"claude-opus-4-6":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25,"tiers":[{"cache_read":1.0,"cache_write":12.5,"input":10,"output":37.5,"tier":{"size":200000}}]},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4.6"},"claude-opus-4-6-think":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25,"tiers":[{"cache_read":1.0,"cache_write":12.5,"input":10,"output":37.5,"tier":{"size":200000}}]},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4.6 Thinking"},"claude-opus-4-7":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25,"tiers":[{"cache_read":1.0,"cache_write":12.5,"input":10,"output":37.5,"tier":{"size":200000}}]},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4.7"},"claude-opus-4-7-think":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25,"tiers":[{"cache_read":1.0,"cache_write":12.5,"input":10,"output":37.5,"tier":{"size":200000}}]},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4.7 Thinking"},"claude-opus-4-8":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Top Claude Opus tier for the hardest reasoning, coding, and long-horizon agents","name":"Claude Opus 4.8"},"claude-opus-4-8-think":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Top Claude Opus tier for the hardest reasoning, coding, and long-horizon agents","name":"Claude Opus 4.8"},"claude-sonnet-4-6":{"cost":{"cache_read":0.3,"cache_write":3.75,"input":3.0,"output":15.0,"tiers":[{"cache_read":0.6,"cache_write":7.5,"input":6.0,"output":22.5,"tier":{"size":200000}}]},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude Sonnet 4.6"},"claude-sonnet-4-6-think":{"cost":{"cache_read":0.3,"cache_write":3.75,"input":3.0,"output":15.0,"tiers":[{"cache_read":0.6,"cache_write":7.5,"input":6.0,"output":22.5,"tier":{"size":200000}}]},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude Sonnet 4.6 Thinking"},"claude-sonnet-5":{"cost":{"cache_read":0.2,"cache_write":2.5,"input":2,"output":10},"description":"Everyday Claude agent model for coding, planning, browsing, and general work","name":"Claude Sonnet 5"},"coding-glm-5.1":{"cost":{"cache_read":0.013,"input":0.06,"output":0.22},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"Coding GLM 5.1"},"coding-glm-5.1-free":{"cost":{"input":0,"output":0},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"Coding GLM 5.1 (free)"},"coding-minimax-m2.7":{"cost":{"input":0.2,"output":0.2},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"Coding MiniMax M2.7"},"coding-minimax-m2.7-free":{"cost":{"input":0,"output":0},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"Coding MiniMax M2.7 (Free)"},"coding-minimax-m2.7-highspeed":{"cost":{"input":0.2,"output":0.2},"description":"High-speed MiniMax model for low-latency coding and agent workflows","name":"Coding MiniMax M2.7 Highspeed"},"coding-xiaomi-mimo-v2.5":{"cost":{"cache_read":0.016,"input":0.08,"output":0.4,"tiers":[{"cache_read":0.032,"input":0.16,"output":0.8,"tier":{"size":256000,"type":"context"}}]},"description":"Open MiMo model for multimodal coding agents and long-context automation","name":"Coding Xiaomi MiMo-V2.5"},"coding-xiaomi-mimo-v2.5-pro":{"cost":{"cache_read":0.04,"input":0.2,"output":0.6,"tiers":[{"cache_read":0.08,"input":0.4,"output":1.2,"tier":{"size":256000,"type":"context"}}]},"description":"Stronger MiMo Pro tier for multimodal reasoning and coding-agent execution","name":"Coding Xiaomi MiMo-V2.5-Pro"},"deep-deepseek-v4-flash":{"cost":{"cache_read":0.0308,"input":0.154,"output":0.308},"description":"Fast DeepSeek model for efficient chat, coding help, and agent loops","name":"DeepSeek V4 Flash (DeepSeek)"},"deep-deepseek-v4-pro":{"cost":{"cache_read":0.004302,"input":0.478,"output":0.956},"description":"Flagship DeepSeek model for coding, reasoning, and agentic work","name":"DeepSeek V4 Pro (DeepSeek)"},"doubao-seed-2-0-code-preview":{"cost":{"cache_read":0.09644,"input":0.48,"output":2.41,"tiers":[{"cache_read":0.144656,"input":0.72,"output":3.62,"tier":{"size":32000}},{"cache_read":0.28932,"input":1.45,"output":7.23,"tier":{"size":128000}}]},"description":"Coding model for repository understanding, refactors, and agentic engineering tasks","name":"Doubao Seed 2.0 Code Preview"},"doubao-seed-2-0-lite-260428":{"cost":{"cache_read":0.01692,"input":0.08,"input_audio":1.269,"output":0.51,"tiers":[{"cache_read":0.02536,"input":0.13,"input_audio":1.902,"output":0.76,"tier":{"size":32000}},{"cache_read":0.05072,"input":0.25,"input_audio":3.804,"output":1.52,"tier":{"size":128000}}]},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Doubao Seed 2.0 Lite 260428"},"doubao-seed-2-0-mini-260428":{"cost":{"cache_read":0.00564,"input":0.03,"input_audio":0.423,"output":0.28,"tiers":[{"cache_read":0.01128,"input":0.06,"input_audio":0.846,"output":0.56,"tier":{"size":32000}},{"cache_read":0.02256,"input":0.11,"input_audio":1.692,"output":1.13,"tier":{"size":128000}}]},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Doubao Seed 2.0 Mini 260428"},"doubao-seed-2-0-pro":{"cost":{"cache_read":0.09644,"input":0.48,"output":2.41,"tiers":[{"cache_read":0.144656,"input":0.72,"output":3.62,"tier":{"size":32000}},{"cache_read":0.28932,"input":1.45,"output":7.23,"tier":{"size":128000}}]},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Doubao Seed 2.0 Pro"},"gemini-2.5-flash":{"cost":{"cache_read":0.03,"input":0.3,"input_audio":1.0,"output":2.5},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini 2.5 Flash"},"gemini-2.5-pro":{"cost":{"cache_read":0.125,"input":1.25,"output":10,"tiers":[{"cache_read":0.25,"input":2.5,"output":15.0,"tier":{"size":200000}}]},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Gemini 2.5 Pro"},"gemini-3-flash-preview":{"cost":{"cache_read":0.05,"input":0.5,"output":3,"tiers":[{"cache_read":0.05,"input":0.5,"output":3.0,"tier":{"size":200000}}]},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini 3 Flash Preview"},"gemini-3.1-flash-lite":{"cost":{"cache_read":0.025,"cache_write":1.0,"input":0.25,"output":1.5},"description":"Low-latency Gemini model for high-volume multimodal and agent workloads","name":"Gemini 3.1 Flash Lite"},"gemini-3.1-pro-preview":{"cost":{"cache_read":0.2,"input":2,"output":12,"tiers":[{"cache_read":0.4,"input":4.0,"output":18.0,"tier":{"size":200000}}]},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Gemini 3.1 Pro Preview"},"gemini-3.1-pro-preview-customtools":{"cost":{"cache_read":0.2,"input":2,"output":12,"tiers":[{"cache_read":0.4,"input":4,"output":18,"tier":{"size":200000}}]},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Gemini 3.1 Pro Preview Custom Tools"},"gemini-3.5-flash":{"cost":{"cache_read":1.5,"input":1.5,"output":9},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini 3.5 Flash"},"glm-5.2":{"cost":{"cache_read":0.2817,"input":1.1268,"output":3.9438},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM-5.2"},"glm-5v-turbo":{"cost":{"cache_read":0.169008,"input":0.7042,"output":3.09848},"description":"GLM vision model for visual reasoning, documents, and multimodal agents","name":"GLM 5 Vision Turbo"},"gpt-5.1":{"cost":{"cache_read":0.13,"input":1.25,"output":10.0},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-5.1"},"gpt-5.1-codex":{"cost":{"cache_read":0.125,"input":1.25,"output":10.0},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.1 Codex"},"gpt-5.1-codex-mini":{"cost":{"cache_read":0.025,"input":0.25,"output":2.0},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.1 Codex mini"},"gpt-5.2":{"cost":{"cache_read":0.175,"input":1.75,"output":14.0},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-5.2"},"gpt-5.2-codex":{"cost":{"cache_read":0.175,"input":1.75,"output":14.0},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.2 Codex"},"gpt-5.3-codex":{"cost":{"cache_read":0.175,"input":1.75,"output":14.0},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.3 Codex"},"gpt-5.4":{"cost":{"cache_read":0.25,"input":2.5,"output":15.0,"tiers":[{"cache_read":0.5,"input":5.0,"output":22.5,"tier":{"size":272000}}]},"description":"Frontier GPT model for professional reasoning, coding, and multimodal work","name":"GPT-5.4"},"gpt-5.4-mini":{"cost":{"cache_read":0.075,"input":0.75,"output":4.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-5.4 mini"},"gpt-5.5":{"cost":{"cache_read":0.5,"input":5.0,"output":30.0,"tiers":[{"cache_read":1.0,"input":10.0,"output":45.0,"tier":{"size":272000}}]},"description":"Frontier GPT model for professional reasoning, coding, and multimodal work","name":"GPT-5.5"},"gpt-5.6-luna":{"cost":{"cache_read":0.1,"cache_write":1.25,"input":1,"output":6},"description":"Cost-efficient GPT-5.6 model for fast, high-volume workloads","name":"GPT-5.6 Luna"},"gpt-5.6-sol":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":30},"description":"Frontier GPT-5.6 model for complex professional work, coding, and agentic workflows","name":"GPT-5.6 Sol"},"gpt-5.6-terra":{"cost":{"cache_read":0.25,"cache_write":3.125,"input":2.5,"output":15},"description":"Balanced GPT-5.6 model for capable, cost-efficient everyday work","name":"GPT-5.6 Terra"},"grok-4.3":{"cost":{"cache_read":0.2,"input":1.25,"output":2.5,"tiers":[{"cache_read":0.4,"input":2.5,"output":5.0,"tier":{"size":200000}}]},"description":"Grok model for agentic tool use, reasoning, coding, and live assistance","name":"Grok 4.3"},"grok-4.5":{"cost":{"cache_read":0.5,"input":2,"output":6},"description":"xAI's latest Grok for chat, coding, agentic tools, and lower hallucination risk","name":"Grok 4.5"},"grok-build-0.1":{"cost":{"cache_read":0.2,"input":1,"output":2},"description":"Fast Grok coding model tuned for agentic engineering and iterative edits","name":"Grok Build 0.1"},"hy3-preview":{"cost":{"cache_read":0.051,"input":0.17,"output":0.566661},"description":"Tencent Hy reasoning model for coding, instruction following, and agent tasks","name":"Hy3 Preview"},"kimi-k2.5":{"cost":{"cache_read":0.1,"input":0.6,"output":3},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi K2.5"},"kimi-k2.6":{"cost":{"cache_read":0.16,"input":0.95,"output":4},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi K2.6"},"kimi-k2.7-code":{"cost":{"cache_read":0.160835,"input":0.95,"output":3.9995},"description":"Coding-focused Kimi model, stronger on long-horizon repo work with less overthinking","name":"Kimi K2.7 Code"},"kimi-k2.7-code-highspeed":{"cost":{"cache_read":0.32167,"input":1.9,"output":7.999},"description":"Lower-latency Kimi Code variant for interactive edits and coding-agent loops","name":"Kimi K2.7 Code Highspeed"},"kimi-k3":{"cost":{"cache_read":0.3,"input":3.0,"output":15.0},"description":"Multimodal Kimi model with 1M context and toggleable max-effort thinking for long-horizon agent work","name":"Kimi K3"},"minimax-m2.7":{"cost":{"cache_read":0.06,"cache_write":0.375,"input":0.3,"output":1.2},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"MiniMax M2.7"},"qwen3.6-flash":{"cost":{"cache_read":0.0169,"cache_write":0.21125,"input":0.17,"output":1.01,"tiers":[{"cache_read":0.0676,"cache_write":0.845,"input":0.68,"output":4.06,"tier":{"size":256000}}]},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Qwen3.6 Flash"},"qwen3.6-max-preview":{"cost":{"cache_read":0.1268,"cache_write":1.585,"input":1.27,"output":7.61,"tiers":[{"cache_read":0.2112,"cache_write":2.64,"input":2.11,"output":12.67,"tier":{"size":128000}}]},"description":"Flagship model for demanding analysis, coding, and production agent workflows","name":"Qwen3.6 Max Preview"},"qwen3.6-plus":{"cost":{"cache_read":0.0282,"cache_write":0.3525,"input":0.28,"output":1.69,"tiers":[{"cache_read":0.1128,"cache_write":1.41,"input":1.13,"output":6.77,"tier":{"size":256000}}]},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Qwen3.6 Plus"},"qwen3.7-max":{"cost":{"cache_read":0.169,"cache_write":2.1125,"input":1.69,"output":5.07},"description":"Qwen frontier model tuned for agent frameworks, coding assistants, and long tasks","name":"Qwen3.7 Max"},"qwen3.7-plus":{"cost":{"cache_read":0.0564,"cache_write":0.3525,"input":0.282,"output":1.128},"description":"Multimodal Qwen workhorse for long-context agents, visual inputs, and coding","name":"Qwen3.7 Plus"},"xiaomi-mimo-v2.5":{"cost":{"cache_read":0.088,"input":0.44,"output":2.2,"tiers":[{"cache_read":0.176,"input":0.88,"output":4.4,"tier":{"size":256000,"type":"context"}}]},"description":"Open MiMo model for multimodal coding agents and long-context automation","name":"Xiaomi MiMo-V2.5"},"xiaomi-mimo-v2.5-free":{"cost":{"cache_read":0,"input":0,"output":0},"description":"Open MiMo model for multimodal coding agents and long-context automation","name":"Xiaomi MiMo-V2.5 (free)"},"xiaomi-mimo-v2.5-pro":{"cost":{"cache_read":0.22,"input":1.1,"output":3.3,"tiers":[{"cache_read":0.44,"input":2.2,"output":6.6,"tier":{"size":256000,"type":"context"}}]},"description":"Stronger MiMo Pro tier for multimodal reasoning and coding-agent execution","name":"Xiaomi MiMo-V2.5-Pro"},"xiaomi-mimo-v2.5-pro-free":{"cost":{"cache_read":0,"input":0,"output":0},"description":"Stronger MiMo Pro tier for multimodal reasoning and coding-agent execution","name":"Xiaomi MiMo-V2.5-Pro (free)"},"zai-glm-5.1":{"cost":{"cache_read":0.183112,"input":0.845,"output":3.38},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM-5.1 (Z.ai)"}}},"anthropic":{"models":{"claude-fable-5":{"cost":{"cache_read":1,"cache_write":12.5,"input":10,"output":50},"description":"Claude model for creative writing, analysis, and controlled agent workflows","name":"Claude Fable 5"},"claude-haiku-4-5":{"cost":{"cache_read":0.1,"cache_write":1.25,"input":1,"output":5},"description":"Fast Claude lane for lightweight agents, office tasks, and responsive chat","name":"Claude Haiku 4.5 (latest)"},"claude-haiku-4-5-20251001":{"cost":{"cache_read":0.1,"cache_write":1.25,"input":1,"output":5},"description":"Fast Claude model for responsive assistance, classification, and lightweight agents","name":"Claude Haiku 4.5"},"claude-opus-4-5":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4.5 (latest)"},"claude-opus-4-5-20251101":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4.5"},"claude-opus-4-6":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"High-end Claude for difficult coding, planning, and slower expert reasoning","name":"Claude Opus 4.6"},"claude-opus-4-7":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Stronger Opus tier for advanced software work and high-stakes reasoning","name":"Claude Opus 4.7"},"claude-opus-4-8":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Top Claude Opus tier for the hardest reasoning, coding, and long-horizon agents","name":"Claude Opus 4.8"},"claude-opus-5":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Strongest Claude Opus model for coding, agents, and professional work","name":"Claude Opus 5"},"claude-sonnet-4-5":{"cost":{"cache_read":0.3,"cache_write":3.75,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude Sonnet 4.5 (latest)"},"claude-sonnet-4-5-20250929":{"cost":{"cache_read":0.3,"cache_write":3.75,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude Sonnet 4.5"},"claude-sonnet-4-6":{"cost":{"cache_read":0.3,"cache_write":3.75,"input":3,"output":15},"description":"Claude workhorse for coding agents, careful analysis, and production cost control","name":"Claude Sonnet 4.6"},"claude-sonnet-5":{"cost":{"cache_read":0.2,"cache_write":2.5,"input":2,"output":10},"description":"Everyday Claude agent model for coding, planning, browsing, and general work","name":"Claude Sonnet 5"}}},"azure_openai":{"models":{"claude-fable-5":{"cost":{"cache_read":1.0,"cache_write":12.5,"input":10.0,"output":50.0},"description":"Claude model for creative writing, analysis, and controlled agent workflows","name":"Claude Fable 5"},"claude-haiku-4-5":{"cost":{"cache_read":0.1,"cache_write":1.25,"input":1.0,"output":5.0},"description":"Fast Claude model for responsive assistance, classification, and lightweight agents","name":"Claude Haiku 4.5"},"claude-opus-4-1":{"cost":{"cache_read":1.5,"cache_write":18.75,"input":15.0,"output":75.0},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4.1"},"claude-opus-4-5":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5.0,"output":25.0},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4.5"},"claude-opus-4-6":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5.0,"output":25.0,"tiers":[{"cache_read":1.0,"cache_write":12.5,"input":10.0,"output":37.5,"tier":{"size":200000}}]},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4.6"},"claude-opus-4-8":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5.0,"output":25.0,"tiers":[{"cache_read":1.0,"cache_write":12.5,"input":10.0,"output":37.5,"tier":{"size":200000}}]},"description":"Top Claude Opus tier for the hardest reasoning, coding, and long-horizon agents","name":"Claude Opus 4.8"},"claude-opus-5":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5.0,"output":25.0},"description":"Strongest Claude Opus model for coding, agents, and professional work","name":"Claude Opus 5"},"claude-sonnet-4-5":{"cost":{"cache_read":0.3,"cache_write":3.75,"input":3.0,"output":15.0},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude Sonnet 4.5"},"claude-sonnet-4-6":{"cost":{"cache_read":0.3,"cache_write":3.75,"input":3,"output":15},"description":"Claude workhorse for coding agents, careful analysis, and production cost control","name":"Claude Sonnet 4.6"},"claude-sonnet-5":{"cost":{"cache_read":0.2,"cache_write":2.5,"input":2.0,"output":10.0},"description":"Everyday Claude agent model for coding, planning, browsing, and general work","name":"Claude Sonnet 5"},"codestral-2501":{"cost":{"input":0.3,"output":0.9},"description":"Mistral coding model for code completion, generation, and developer workflows","name":"Codestral 25.01"},"codex-mini":{"cost":{"cache_read":0.375,"input":1.5,"output":6.0},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"Codex Mini"},"cohere-command-a":{"cost":{"input":2.5,"output":10.0},"description":"Cohere command model for multilingual enterprise agents, tools, and chat","name":"Command A"},"cohere-embed-v-4-0":{"cost":{"input":0.12,"output":0.0},"description":"Embedding model for semantic search, retrieval, clustering, and ranking pipelines","name":"Embed v4"},"cohere-embed-v3-english":{"cost":{"input":0.1,"output":0.0},"description":"Embedding model for semantic search, retrieval, clustering, and ranking pipelines","name":"Embed v3 English"},"cohere-embed-v3-multilingual":{"cost":{"input":0.1,"output":0.0},"description":"Embedding model for semantic search, retrieval, clustering, and ranking pipelines","name":"Embed v3 Multilingual"},"deepseek-r1":{"cost":{"input":1.35,"output":5.4},"description":"DeepSeek reasoning model for multi-step analysis, math, coding, and tools","name":"DeepSeek-R1"},"deepseek-v3.2":{"cost":{"input":0.58,"output":1.68},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek-V3.2"},"deepseek-v3.2-speciale":{"cost":{"input":0.58,"output":1.68},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek-V3.2-Speciale"},"deepseek-v4-flash":{"cost":{"input":0.19,"output":0.51},"description":"Fast DeepSeek V4 lane for economical reasoning, coding, and long-context work","name":"DeepSeek-V4-Flash"},"deepseek-v4-pro":{"cost":{"input":1.74,"output":3.48},"description":"Open MoE flagship with million-token context for coding and long agent runs","name":"DeepSeek-V4-Pro"},"gpt-3.5-turbo-0125":{"cost":{"input":0.5,"output":1.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-3.5 Turbo 0125"},"gpt-3.5-turbo-1106":{"cost":{"input":1.0,"output":2.0},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-3.5 Turbo 1106"},"gpt-3.5-turbo-instruct":{"cost":{"input":1.5,"output":2.0},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-3.5 Turbo Instruct"},"gpt-4-turbo":{"cost":{"input":10,"output":30},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-4 Turbo"},"gpt-4-turbo-vision":{"cost":{"input":10.0,"output":30.0},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-4 Turbo Vision"},"gpt-4.1":{"cost":{"cache_read":0.5,"input":2,"output":8},"description":"Long-lived GPT workhorse for coding, instruction following, and production apps","name":"GPT-4.1"},"gpt-4.1-mini":{"cost":{"cache_read":0.1,"input":0.4,"output":1.6},"description":"Affordable GPT-4.1 lane for fast coding help and structured extraction","name":"GPT-4.1 mini"},"gpt-4.1-nano":{"cost":{"cache_read":0.025,"input":0.1,"output":0.4},"description":"Tiny GPT-4.1 option for classification, routing, and very high-volume tasks","name":"GPT-4.1 nano"},"gpt-4o":{"cost":{"cache_read":1.25,"input":2.5,"output":10},"description":"Omni-era GPT for multimodal chat, practical coding, and general assistants","name":"GPT-4o"},"gpt-4o-mini":{"cost":{"cache_read":0.075,"input":0.15,"output":0.6},"description":"Small omni GPT for cheap multimodal assistance and production-scale traffic","name":"GPT-4o mini"},"gpt-5":{"cost":{"cache_read":0.13,"input":1.25,"output":10.0},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-5"},"gpt-5-codex":{"cost":{"cache_read":0.13,"input":1.25,"output":10},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5-Codex"},"gpt-5-mini":{"cost":{"cache_read":0.03,"input":0.25,"output":2.0},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-5 Mini"},"gpt-5-nano":{"cost":{"cache_read":0.01,"input":0.05,"output":0.4},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-5 Nano"},"gpt-5-pro":{"cost":{"input":15,"output":120},"description":"Higher-accuracy GPT-5 tier for tough analysis, coding reviews, and planning","name":"GPT-5 Pro"},"gpt-5.1":{"cost":{"cache_read":0.125,"input":1.25,"output":10.0},"description":"Speech generation model for controllable voice, narration, and audio delivery","name":"GPT-5.1"},"gpt-5.1-codex":{"cost":{"cache_read":0.125,"input":1.25,"output":10.0},"description":"Speech generation model for controllable voice, narration, and audio delivery","name":"GPT-5.1 Codex"},"gpt-5.1-codex-max":{"cost":{"cache_read":0.125,"input":1.25,"output":10},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.1 Codex Max"},"gpt-5.1-codex-mini":{"cost":{"cache_read":0.025,"input":0.25,"output":2.0},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.1 Codex Mini"},"gpt-5.2":{"cost":{"cache_read":0.125,"input":1.75,"output":14.0},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-5.2"},"gpt-5.2-codex":{"cost":{"cache_read":0.175,"input":1.75,"output":14.0},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.2 Codex"},"gpt-5.3-codex":{"cost":{"cache_read":0.175,"input":1.75,"output":14.0},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.3 Codex"},"gpt-5.4":{"cost":{"cache_read":0.25,"input":2.5,"output":15,"tiers":[{"cache_read":0.5,"input":5,"output":22.5,"tier":{"size":272000,"type":"context"}}]},"description":"Agent-ready GPT for coding and computer-use workflows at a lower cost","name":"GPT-5.4"},"gpt-5.4-mini":{"cost":{"cache_read":0.075,"input":0.75,"output":4.5},"description":"Strong small GPT for coding subagents, quick tool use, and high-volume work","name":"GPT-5.4 Mini"},"gpt-5.4-nano":{"cost":{"cache_read":0.02,"input":0.2,"output":1.25},"description":"Cheapest GPT-5.4 lane for simple routing, extraction, and bulk automation","name":"GPT-5.4 Nano"},"gpt-5.4-pro":{"cost":{"input":30,"output":180,"tiers":[{"input":60,"output":270,"tier":{"size":272000,"type":"context"}}]},"description":"More exact GPT-5.4 tier for demanding professional reasoning and agent tasks","name":"GPT-5.4 Pro"},"gpt-5.5":{"cost":{"cache_read":0.5,"input":5,"output":30,"tiers":[{"cache_read":1,"input":10,"output":45,"tier":{"size":272000,"type":"context"}}]},"description":"Default frontier GPT for coding, computer use, research, and knowledge work","name":"GPT-5.5"},"gpt-5.6-luna":{"cost":{"cache_read":0.1,"cache_write":1.25,"input":1,"output":6,"tiers":[{"cache_read":0.2,"cache_write":2.5,"input":2,"output":9,"tier":{"size":272000,"type":"context"}}]},"description":"Cost-efficient GPT-5.6 model for fast, high-volume workloads","name":"GPT-5.6 Luna"},"gpt-5.6-sol":{"cost":{"cache_read":0.5,"input":5,"output":30,"tiers":[{"cache_read":1,"input":10,"output":45,"tier":{"size":272000,"type":"context"}}]},"description":"Frontier GPT-5.6 model for complex professional work, coding, and agentic workflows","name":"GPT-5.6 Sol"},"gpt-5.6-terra":{"cost":{"cache_read":0.25,"cache_write":3.125,"input":2.5,"output":15,"tiers":[{"cache_read":0.5,"cache_write":6.25,"input":5,"output":22.5,"tier":{"size":272000,"type":"context"}}]},"description":"Balanced GPT-5.6 model for capable, cost-efficient everyday work","name":"GPT-5.6 Terra"},"gpt-chat-latest":{"cost":{"cache_read":0.5,"input":5,"output":30},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT Chat Latest"},"gpt-image-1":{"cost":{"cache_read":1.25,"input":5.0,"output":40.0},"description":"OpenAI image model for production generation, edits, and brand-safe visual workflows","name":"GPT-Image-1"},"gpt-image-1.5":{"cost":{"cache_read":1.25,"input":5.0,"output":32.0},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"GPT-Image-1.5"},"gpt-image-2":{"cost":{"cache_read":1.25,"input":5.0,"output":30.0},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"GPT-Image-2"},"grok-4-1-fast-non-reasoning":{"cost":{"cache_read":0.05,"input":0.2,"output":0.5},"description":"Fast Grok model for responsive chat, reasoning, and tool-assisted work","name":"Grok 4.1 Fast (Non-Reasoning)"},"grok-4-1-fast-reasoning":{"cost":{"cache_read":0.05,"input":0.2,"output":0.5},"description":"Fast Grok model for responsive chat, reasoning, and tool-assisted work","name":"Grok 4.1 Fast (Reasoning)"},"grok-4-20-non-reasoning":{"cost":{"input":2.0,"output":6.0},"description":"Grok model for agentic tool use, reasoning, coding, and live assistance","name":"Grok 4.20 (Non-Reasoning)"},"grok-4-20-reasoning":{"cost":{"input":2.0,"output":6.0},"description":"Grok model for agentic tool use, reasoning, coding, and live assistance","name":"Grok 4.20 (Reasoning)"},"kimi-k2.5":{"cost":{"input":0.6,"output":3.0},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi K2.5"},"kimi-k2.6":{"cost":{"input":0.95,"output":4.0},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi K2.6"},"kimi-k2.7-code":{"cost":{"cache_read":0.19,"input":0.95,"output":4.0},"description":"Coding-focused Kimi model, stronger on long-horizon repo work with less overthinking","name":"Kimi K2.7 Code"},"llama-3.3-70b-instruct":{"cost":{"input":0.71,"output":0.71},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama-3.3-70B-Instruct"},"llama-4-maverick-17b-128e-instruct-fp8":{"cost":{"input":0.25,"output":1.0},"description":"Open multimodal Llama model for strong reasoning and fast responses","name":"Llama 4 Maverick 17B 128E Instruct FP8"},"llama-4-scout-17b-16e-instruct":{"cost":{"input":0.2,"output":0.78},"description":"Open multimodal Llama model for long-context analysis and efficient agents","name":"Llama 4 Scout 17B 16E Instruct"},"ministral-3b":{"cost":{"input":0.04,"output":0.04},"description":"Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads","name":"Ministral 3B"},"mistral-medium-2505":{"cost":{"input":0.4,"output":2.0},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral Medium 3"},"mistral-small-2503":{"cost":{"input":0.1,"output":0.3},"description":"Efficient Mistral model for fast chat, extraction, and production assistants","name":"Mistral Small 3.1"},"model-router":{"cost":{"input":0.14,"output":0.0},"description":"Automatic model router for matching prompts to suitable backends and budgets","name":"Model Router"},"o1":{"cost":{"cache_read":7.5,"input":15.0,"output":60.0},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"o1"},"o3":{"cost":{"cache_read":0.5,"input":2,"output":8},"description":"Deliberate o-series reasoner for hard math, coding, and multi-step analysis","name":"o3"},"o3-mini":{"cost":{"cache_read":0.55,"input":1.1,"output":4.4},"description":"Smaller o-series reasoner for economical coding, math, and planning tasks","name":"o3-mini"},"o4-mini":{"cost":{"cache_read":0.275,"input":1.1,"output":4.4},"description":"Fast o-series model for compact reasoning, coding, and tool use","name":"o4-mini"},"phi-4":{"cost":{"input":0.125,"output":0.5},"description":"Open-weight instruction model for adaptable chat and self-hosted production workloads","name":"Phi-4"},"phi-4-mini":{"cost":{"input":0.075,"output":0.3},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Phi-4-mini"},"phi-4-mini-reasoning":{"cost":{"input":0.075,"output":0.3},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Phi-4-mini-reasoning"},"phi-4-multimodal":{"cost":{"input":0.08,"input_audio":4.0,"output":0.32},"description":"Multimodal model for analyzing text, images, documents, and rich media","name":"Phi-4-multimodal"},"phi-4-reasoning":{"cost":{"input":0.125,"output":0.5},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"Phi-4-reasoning"},"phi-4-reasoning-plus":{"cost":{"input":0.125,"output":0.5},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"Phi-4-reasoning-plus"},"text-embedding-3-large":{"cost":{"input":0.13,"output":0.0},"description":"Embedding model for semantic search, retrieval, clustering, and ranking pipelines","name":"text-embedding-3-large"},"text-embedding-3-small":{"cost":{"input":0.02,"output":0.0},"description":"Embedding model for semantic search, retrieval, clustering, and ranking pipelines","name":"text-embedding-3-small"},"text-embedding-ada-002":{"cost":{"input":0.1,"output":0.0},"description":"Embedding model for semantic search, retrieval, clustering, and ranking pipelines","name":"text-embedding-ada-002"}}},"baseten":{"models":{"MiniMaxAI/MiniMax-M2.5":{"cost":{"input":0.3,"output":1.2},"description":"Legacy model retained for compatibility with older integrations","name":"MiniMax-M2.5"},"deepseek-ai/DeepSeek-V3.1":{"cost":{"input":0.5,"output":1.5},"description":"Legacy model retained for compatibility with older integrations","name":"DeepSeek V3.1"},"deepseek-ai/DeepSeek-V4-Flash-0731":{"cost":{"cache_read":0.028,"input":0.13,"output":0.26},"description":"Official DeepSeek V4 Flash release with enhanced agentic capabilities and integrated DSpark speculative decoding","name":"Deepseek V4 Flash 0731"},"deepseek-ai/DeepSeek-V4-Pro":{"cost":{"cache_read":0.145,"input":1.74,"output":3.48},"description":"Open MoE flagship with million-token context for coding and long agent runs","name":"Deepseek V4 Pro"},"moonshotai/Kimi-K2.5":{"cost":{"cache_read":0.12,"input":0.6,"output":3},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi K2.5"},"moonshotai/Kimi-K2.6":{"cost":{"cache_read":0.16,"input":0.95,"output":4},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi K2.6"},"moonshotai/Kimi-K2.7-Code":{"cost":{"cache_read":0.16,"input":0.95,"output":4},"description":"Coding-focused Kimi model, stronger on long-horizon repo work with less overthinking","name":"Kimi K2.7 Code"},"moonshotai/Kimi-K3":{"cost":{"input":3,"output":15},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi K3"},"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B":{"cost":{"cache_read":0.12,"input":0.6,"output":2.4},"description":"Largest Nemotron 3 model for maximum open-weight reasoning and agent accuracy","name":"Nemotron Ultra"},"nvidia/Nemotron-120B-A12B":{"cost":{"cache_read":0.06,"input":0.3,"output":0.75},"description":"Nemotron middle tier for collaborative agents and high-volume reasoning workloads","name":"Nemotron Super"},"openai/gpt-oss-120b":{"cost":{"input":0.1,"output":0.5},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"OpenAI GPT 120B"},"thinkingmachines/inkling":{"cost":{"input":1,"output":4.05},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Inkling"},"thinkingmachines/inkling-small":{"cost":{"cache_read":0.1,"input":0.5,"output":1.2},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Inkling Small"},"zai-org/GLM-4.7":{"cost":{"cache_read":0.12,"input":0.6,"output":2.2},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM 4.7"},"zai-org/GLM-5":{"cost":{"cache_read":0.2,"input":0.95,"output":3.15},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM 5"},"zai-org/GLM-5.1":{"cost":{"cache_read":0.26,"input":1.3,"output":4.3},"description":"Strong GLM coding model for agentic engineering, terminals, and repository generation","name":"GLM 5.1"},"zai-org/GLM-5.2":{"cost":{"cache_read":0.3,"input":1.4,"output":4.4},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM 5.2"},"zai-org/GLM-5.2-Fast":{"cost":{"cache_read":0.21,"input":2.1,"output":6.6},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM 5.2 Fast"}}},"cerebras":{"models":{"gemma-4-31b":{"cost":{"input":0.99,"output":1.49},"description":"Largest Gemma 4 instruction model for open, self-hosted chat and reasoning","name":"Gemma 4 31B IT"},"gpt-oss-120b":{"cost":{"input":0.35,"output":0.75},"description":"Open GPT reasoning model for self-hosted agents and controllable deployments","name":"GPT OSS 120B"},"zai-glm-4.7":{"cost":{"cache_read":2.25,"cache_write":0,"input":2.25,"output":2.75},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"Z.AI GLM-4.7"}}},"chutes":{"models":{"Nemotron-3-Nano-Omni-30B-TEE":{"cost":{"cache_read":0.0024499999999999995,"input":0.0245,"output":0.0978},"description":"Omni-modal model for text, vision, audio, and multimodal agent tasks","name":"Nemotron 3 Nano Omni 30B TEE"},"Qwen/Qwen3-235B-A22B-Thinking-2507-TEE":{"cost":{"cache_read":0.029889999999999993,"input":0.2989,"output":1.1957},"description":"Qwen reasoning model for deliberate problem solving, math, and coding","name":"Qwen3 235B A22B Thinking 2507 TEE"},"Qwen/Qwen3-32B-TEE":{"cost":{"cache_read":0.010399999999999998,"input":0.104,"output":0.416},"description":"Dense open Qwen model for self-hosted chat, reasoning, and coding","name":"Qwen3 32B TEE"},"Qwen/Qwen3.5-397B-A17B-TEE":{"cost":{"cache_read":0.04499999999999999,"input":0.45,"output":3},"description":"Large open Qwen multimodal MoE for visual agents and long technical tasks","name":"Qwen3.5 397B A17B TEE"},"Qwen/Qwen3.6-27B-TEE":{"cost":{"cache_read":0.029999999999999992,"input":0.3,"output":2},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.6 27B TEE"},"deepseek-ai/DeepSeek-V3.2-TEE":{"cost":{"cache_read":0.09999999999999998,"input":1,"output":1},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek V3.2 TEE"},"deepseek-ai/DeepSeek-V4-Flash-0731-TEE":{"cost":{"cache_read":0.013999999999999999,"input":0.14,"output":0.28},"description":"Official DeepSeek V4 Flash release with enhanced agentic capabilities and integrated DSpark speculative decoding","name":"DeepSeek V4 Flash 0731 TEE"},"google/gemma-4-31B-turbo-TEE":{"cost":{"cache_read":0.011999999999999997,"input":0.12,"output":0.37},"description":"Largest Gemma 4 instruction model for open, self-hosted chat and reasoning","name":"gemma 4 31B turbo TEE"},"moonshotai/Kimi-K2.6-TEE":{"cost":{"cache_read":0.05799999999999998,"input":0.58,"output":3.4},"description":"Multimodal Kimi workhorse for agent loops, coding tasks, and visual context","name":"Kimi K2.6 TEE"},"moonshotai/Kimi-K3-TEE":{"cost":{"cache_read":0.29999999999999993,"input":3,"output":15},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi K3 TEE"},"unsloth/Mistral-Nemo-Instruct-2407-TEE":{"cost":{"cache_read":0.0024499999999999995,"input":0.0245,"output":0.0978},"description":"Efficient Mistral-NVIDIA open model for multilingual chat and local deployment","name":"Mistral Nemo Instruct 2407 TEE"},"zai-org/GLM-5.1-TEE":{"cost":{"cache_read":0.09799999999999998,"input":0.98,"output":3.08},"description":"Strong GLM coding model for agentic engineering, terminals, and repository generation","name":"GLM 5.1 TEE"},"zai-org/GLM-5.2-TEE":{"cost":{"cache_read":0.12499999999999997,"input":1.25,"output":3.95},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM 5.2 TEE"}}},"clarifai":{"models":{"arcee_ai/AFM/models/trinity-mini":{"cost":{"input":0.045,"output":0.15},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Trinity Mini"},"clarifai/main/models/mm-poly-8b":{"cost":{"input":0.658,"output":1.11},"description":"Multimodal model for analyzing text, images, documents, and rich media","name":"MM Poly 8B"},"deepseek-ai/deepseek-ocr/models/DeepSeek-OCR":{"cost":{"input":0.2,"output":0.7},"description":"OCR model for extracting structured text from documents and screenshots","name":"DeepSeek OCR"},"minimaxai/chat-completion/models/MiniMax-M2_5-high-throughput":{"cost":{"input":0.3,"output":1.2},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"MiniMax-M2.5 High Throughput"},"mistralai/completion/models/Ministral-3-14B-Reasoning-2512":{"cost":{"input":2.5,"output":1.7},"description":"Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads","name":"Ministral 3 14B Reasoning 2512"},"mistralai/completion/models/Ministral-3-3B-Reasoning-2512":{"cost":{"input":1.039,"output":0.54825},"description":"Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads","name":"Ministral 3 3B Reasoning 2512"},"moonshotai/chat-completion/models/Kimi-K2_6":{"cost":{"input":0.95,"output":4},"description":"Multimodal Kimi workhorse for agent loops, coding tasks, and visual context","name":"Kimi K2.6"},"openai/chat-completion/models/gpt-oss-120b-high-throughput":{"cost":{"input":0.09,"output":0.36},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"GPT OSS 120B High Throughput"},"openai/chat-completion/models/gpt-oss-20b":{"cost":{"input":0.045,"output":0.18},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"GPT OSS 20B"},"qwen/qwenCoder/models/Qwen3-Coder-30B-A3B-Instruct":{"cost":{"input":0.11458,"output":0.74812},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen3 Coder 30B A3B Instruct"},"qwen/qwenLM/models/Qwen3-30B-A3B-Instruct-2507":{"cost":{"input":0.3,"output":0.5},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3 30B A3B Instruct 2507"},"qwen/qwenLM/models/Qwen3-30B-A3B-Thinking-2507":{"cost":{"input":0.36,"output":1.3},"description":"Qwen reasoning model for deliberate problem solving, math, and coding","name":"Qwen3 30B A3B Thinking 2507"}}},"cohere":{"models":{"c4ai-aya-expanse-32b":{"description":"Open multilingual model optimized for generation across 23 languages","name":"Aya Expanse 32B"},"c4ai-aya-expanse-8b":{"description":"Compact open multilingual model optimized for generation across 23 languages","name":"Aya Expanse 8B"},"c4ai-aya-vision-32b":{"description":"Open multilingual vision model for OCR, visual reasoning, and image question answering","name":"Aya Vision 32B"},"c4ai-aya-vision-8b":{"description":"Compact open multilingual vision model for OCR and visual question answering","name":"Aya Vision 8B"},"command-a-03-2025":{"cost":{"input":2.5,"output":10.0},"description":"Cohere command model for multilingual enterprise agents, tools, and chat","name":"Command A"},"command-a-plus-05-2026":{"cost":{"input":2.5,"output":10.0},"description":"Cohere's stronger command model for multilingual agents and enterprise workflows","name":"Command A Plus"},"command-a-reasoning-08-2025":{"cost":{"input":2.5,"output":10.0},"description":"Cohere reasoning model for multilingual enterprise agents, tools, and complex workflows","name":"Command A Reasoning"},"command-a-translate-08-2025":{"cost":{"input":2.5,"output":10.0},"description":"Translation model for multilingual conversion, localization, and cross-language workflows","name":"Command A Translate"},"command-a-vision-07-2025":{"cost":{"input":2.5,"output":10.0},"description":"Cohere vision model for multilingual document analysis, OCR, and image understanding","name":"Command A Vision"},"command-r-08-2024":{"cost":{"input":0.15,"output":0.6},"description":"Cohere retrieval model for long-context chat and enterprise RAG workflows","name":"Command R"},"command-r-plus-08-2024":{"cost":{"input":2.5,"output":10.0},"description":"Cohere's RAG workhorse for long-context enterprise search and tool use","name":"Command R+"},"command-r7b-12-2024":{"cost":{"input":0.0375,"output":0.15},"description":"Cohere retrieval model for long-context chat and enterprise RAG workflows","name":"Command R7B"},"command-r7b-arabic-02-2025":{"cost":{"input":0.0375,"output":0.15},"description":"Open Command R model optimized for Arabic enterprise chat, RAG, and cultural knowledge","name":"Command R7B Arabic"},"north-mini-code-1-0":{"cost":{"input":0.0,"output":0.0},"description":"Cohere coding model for practical software engineering and agentic edits","name":"North Mini Code"}}},"dashscope":{"models":{"qvq-max":{"cost":{"input":1.2,"output":4.8},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"QVQ Max"},"qwen-flash":{"cost":{"input":0.05,"output":0.4},"description":"Efficient Qwen model for fast chat, extraction, and high-volume workloads","name":"Qwen Flash"},"qwen-max":{"cost":{"input":1.6,"output":6.4},"description":"Flagship Qwen model for complex reasoning, coding, and agentic workflows","name":"Qwen Max"},"qwen-mt-plus":{"cost":{"input":2.46,"output":7.37},"description":"Translation model for multilingual conversion, localization, and cross-language workflows","name":"Qwen-MT Plus"},"qwen-mt-turbo":{"cost":{"input":0.16,"output":0.49},"description":"Translation model for multilingual conversion, localization, and cross-language workflows","name":"Qwen-MT Turbo"},"qwen-omni-turbo":{"cost":{"input":0.07,"input_audio":4.44,"output":0.27,"output_audio":8.89},"description":"Qwen omni model for text, vision, audio, and multimodal agent tasks","name":"Qwen-Omni Turbo"},"qwen-omni-turbo-realtime":{"cost":{"input":0.27,"input_audio":4.44,"output":1.07,"output_audio":8.89},"description":"Qwen omni model for text, vision, audio, and multimodal agent tasks","name":"Qwen-Omni Turbo Realtime"},"qwen-plus":{"cost":{"input":0.4,"output":1.2,"reasoning":4},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen Plus"},"qwen-plus-character-ja":{"cost":{"input":0.5,"output":1.4},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen Plus Character (Japanese)"},"qwen-turbo":{"cost":{"input":0.05,"output":0.2,"reasoning":0.5},"description":"Efficient Qwen model for fast chat, extraction, and high-volume workloads","name":"Qwen Turbo"},"qwen-vl-max":{"cost":{"input":0.8,"output":3.2},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen-VL Max"},"qwen-vl-ocr":{"cost":{"input":0.72,"output":0.72},"description":"OCR model for extracting structured text from documents and screenshots","name":"Qwen-VL OCR"},"qwen-vl-plus":{"cost":{"input":0.21,"output":0.63},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen-VL Plus"},"qwen2-5-14b-instruct":{"cost":{"input":0.35,"output":1.4},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen2.5 14B Instruct"},"qwen2-5-32b-instruct":{"cost":{"input":0.7,"output":2.8},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen2.5 32B Instruct"},"qwen2-5-72b-instruct":{"cost":{"input":1.4,"output":5.6},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen2.5 72B Instruct"},"qwen2-5-7b-instruct":{"cost":{"input":0.175,"output":0.7},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen2.5 7B Instruct"},"qwen2-5-omni-7b":{"cost":{"input":0.1,"input_audio":6.76,"output":0.4},"description":"Qwen omni model for text, vision, audio, and multimodal agent tasks","name":"Qwen2.5-Omni 7B"},"qwen2-5-vl-72b-instruct":{"cost":{"input":2.8,"output":8.4},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen2.5-VL 72B Instruct"},"qwen2-5-vl-7b-instruct":{"cost":{"input":0.35,"output":1.05},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen2.5-VL 7B Instruct"},"qwen3-14b":{"cost":{"input":0.35,"output":1.4,"reasoning":4.2},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3 14B"},"qwen3-235b-a22b":{"cost":{"input":0.7,"output":2.8,"reasoning":8.4},"description":"Large open Qwen MoE for multilingual reasoning, coding, and tool use","name":"Qwen3 235B-A22B"},"qwen3-32b":{"cost":{"input":0.7,"output":2.8,"reasoning":8.4},"description":"Dense open Qwen model for self-hosted chat, reasoning, and coding","name":"Qwen3 32B"},"qwen3-8b":{"cost":{"input":0.18,"output":0.7,"reasoning":2.1},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3 8B"},"qwen3-asr-flash":{"cost":{"input":0.035,"output":0.035},"description":"Speech transcription model for accurate audio-to-text and captioning workflows","name":"Qwen3-ASR Flash"},"qwen3-coder-30b-a3b-instruct":{"cost":{"input":0.45,"output":2.25},"description":"Smaller Qwen coder for efficient local agents and repo-level fixes","name":"Qwen3-Coder 30B-A3B Instruct"},"qwen3-coder-480b-a35b-instruct":{"cost":{"input":1.5,"output":7.5},"description":"Open Qwen coding heavyweight for repository reasoning and agentic engineering","name":"Qwen3-Coder 480B-A35B Instruct"},"qwen3-coder-flash":{"cost":{"input":0.3,"output":1.5},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen3 Coder Flash"},"qwen3-coder-plus":{"cost":{"input":1,"output":5},"description":"Hosted Qwen coder for software agents, repo edits, and long-context code","name":"Qwen3 Coder Plus"},"qwen3-livetranslate-flash-realtime":{"cost":{"input":10,"input_audio":10,"output":10,"output_audio":38},"description":"Speech generation model for controllable voice, narration, and audio delivery","name":"Qwen3-LiveTranslate Flash Realtime"},"qwen3-max":{"cost":{"input":1.2,"output":6},"description":"Flagship Qwen3 model for coding agents, complex reasoning, and tool use","name":"Qwen3 Max"},"qwen3-next-80b-a3b-instruct":{"cost":{"input":0.5,"output":2},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3-Next 80B-A3B Instruct"},"qwen3-next-80b-a3b-thinking":{"cost":{"input":0.5,"output":6},"description":"Efficient Qwen thinking model for local reasoning, math, and coding agents","name":"Qwen3-Next 80B-A3B (Thinking)"},"qwen3-omni-flash":{"cost":{"input":0.43,"input_audio":3.81,"output":1.66,"output_audio":15.11},"description":"Qwen omni model for text, vision, audio, and multimodal agent tasks","name":"Qwen3-Omni Flash"},"qwen3-omni-flash-realtime":{"cost":{"input":0.52,"input_audio":4.57,"output":1.99,"output_audio":18.13},"description":"Qwen omni model for text, vision, audio, and multimodal agent tasks","name":"Qwen3-Omni Flash Realtime"},"qwen3-vl-235b-a22b":{"cost":{"input":0.7,"output":2.8,"reasoning":8.4},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3-VL 235B-A22B"},"qwen3-vl-30b-a3b":{"cost":{"input":0.2,"output":0.8,"reasoning":2.4},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3-VL 30B-A3B"},"qwen3-vl-plus":{"cost":{"input":0.2,"output":1.6,"reasoning":4.8},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3-VL Plus"},"qwen3.5-122b-a10b":{"cost":{"input":0.4,"output":3.2},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 122B-A10B"},"qwen3.5-27b":{"cost":{"input":0.3,"output":2.4},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 27B"},"qwen3.5-35b-a3b":{"cost":{"input":0.25,"output":2},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 35B-A3B"},"qwen3.5-397b-a17b":{"cost":{"input":0.6,"output":3.6},"description":"Large open Qwen multimodal MoE for visual agents and long technical tasks","name":"Qwen3.5 397B-A17B"},"qwen3.5-plus":{"cost":{"input":0.4,"output":2.4,"reasoning":2.4},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 Plus"},"qwen3.6-27b":{"cost":{"input":0.6,"output":3.6},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.6 27B"},"qwen3.6-35b-a3b":{"cost":{"input":0.248,"output":1.485},"description":"Open multimodal Qwen MoE for local agents that need vision, audio, and code","name":"Qwen3.6 35B-A3B"},"qwen3.6-flash":{"cost":{"cache_write":0.234375,"input":0.1875,"output":1.125},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.6 Flash"},"qwen3.6-max-preview":{"cost":{"cache_read":0.13,"cache_write":1.625,"input":1.3,"output":7.8},"description":"Flagship Qwen model for complex reasoning, coding, and agentic workflows","name":"Qwen3.6 Max Preview"},"qwen3.6-plus":{"cost":{"cache_read":0.05,"cache_write":0.625,"input":0.5,"output":3.0,"tiers":[{"cache_read":0.2,"cache_write":2.5,"input":2.0,"output":6.0,"tier":{"size":256000}}]},"description":"Earlier Qwen multimodal workhorse for million-token agent and document tasks","name":"Qwen3.6 Plus"},"qwen3.7-max":{"cost":{"cache_read":0.5,"cache_write":3.125,"input":2.5,"output":7.5},"description":"Qwen frontier model tuned for agent frameworks, coding assistants, and long tasks","name":"Qwen3.7 Max"},"qwen3.7-plus":{"cost":{"cache_read":0.05,"cache_write":0.625,"input":0.5,"output":3.0,"tiers":[{"cache_read":0.2,"cache_write":2.5,"input":2.0,"output":6.0,"tier":{"size":256000}}]},"description":"Multimodal Qwen workhorse for long-context agents, visual inputs, and coding","name":"Qwen3.7 Plus"},"qwen3.8-max":{"cost":{"cache_read":0.25,"cache_write":2.5,"input":2.0,"output":6.0},"description":"2.4-trillion-parameter MoE flagship for coding, professional work, multimodal understanding, and long-horizon agentic workflows","name":"Qwen3.8 Max"},"qwq-plus":{"cost":{"input":0.8,"output":2.4},"description":"Qwen reasoning model for deliberate problem solving, math, and coding","name":"QwQ Plus"}}},"databricks":{"models":{"databricks-claude-haiku-4-5":{"cost":{"cache_read":0.1,"cache_write":1.25,"input":1,"output":5},"description":"Fast Claude lane for lightweight agents, office tasks, and responsive chat","name":"Claude Haiku 4.5 (latest)"},"databricks-claude-opus-4-1":{"cost":{"cache_read":1.5,"cache_write":18.75,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4.1 (latest)"},"databricks-claude-opus-4-5":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4.5 (latest)"},"databricks-claude-opus-4-6":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"High-end Claude for difficult coding, planning, and slower expert reasoning","name":"Claude Opus 4.6"},"databricks-claude-opus-4-7":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Stronger Opus tier for advanced software work and high-stakes reasoning","name":"Claude Opus 4.7"},"databricks-claude-sonnet-4":{"cost":{"cache_read":0.3,"cache_write":3.75,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude Sonnet 4.5"},"databricks-claude-sonnet-4-5":{"cost":{"cache_read":0.3,"cache_write":3.75,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude Sonnet 4.5 (latest)"},"databricks-claude-sonnet-4-6":{"cost":{"cache_read":0.3,"cache_write":3.75,"input":3,"output":15},"description":"Claude workhorse for coding agents, careful analysis, and production cost control","name":"Claude Sonnet 4.6"},"databricks-gemini-2-5-flash":{"cost":{"cache_read":0.03,"input":0.3,"input_audio":1,"output":2.5},"description":"Fast Gemini workhorse for multimodal apps where latency and price matter","name":"Gemini 2.5 Flash"},"databricks-gemini-2-5-pro":{"cost":{"cache_read":0.125,"input":1.25,"output":10,"tiers":[{"cache_read":0.25,"input":2.5,"output":15,"tier":{"size":200000,"type":"context"}}]},"description":"Google's proven reasoning model for coding, math, and multimodal analysis","name":"Gemini 2.5 Pro"},"databricks-gemini-3-1-flash-lite":{"cost":{"cache_read":0.025,"input":0.25,"input_audio":0.5,"output":1.5},"description":"Low-latency Gemini model for high-volume multimodal and agent workloads","name":"Gemini 3.1 Flash Lite Preview"},"databricks-gemini-3-1-pro":{"cost":{"cache_read":0.2,"input":2,"output":12,"tiers":[{"cache_read":0.4,"input":4,"output":18,"tier":{"size":200000,"type":"context"}}]},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Gemini 3.1 Pro Preview Custom Tools"},"databricks-gemini-3-flash":{"cost":{"cache_read":0.05,"input":0.5,"input_audio":1,"output":3},"description":"New Gemini flash lane bringing frontier-style multimodal reasoning to cheaper runs","name":"Gemini 3 Flash Preview"},"databricks-gemini-3-pro":{"cost":{"cache_read":0.2,"input":2,"output":12,"tiers":[{"cache_read":0.4,"input":4,"output":18,"tier":{"size":200000,"type":"context"}}]},"description":"Preview Gemini flagship for complex reasoning, coding, and rich multimodal prompts","name":"Gemini 3 Pro Preview"},"databricks-glm-5-2":{"cost":{"cache_read":0.26,"input":1.4,"output":4.4},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM-5.2"},"databricks-gpt-5":{"cost":{"cache_read":0.125,"input":1.25,"output":10},"description":"Original GPT-5 workhorse for reasoning, coding, writing, and tool workflows","name":"GPT-5"},"databricks-gpt-5-1":{"cost":{"cache_read":0.125,"input":1.25,"output":10},"description":"Sharper GPT-5 generation for coding, product work, and tool-assisted tasks","name":"GPT-5.1"},"databricks-gpt-5-2":{"cost":{"cache_read":0.175,"input":1.75,"output":14},"description":"Reliable GPT generation for broad coding, writing, and tool-assisted product work","name":"GPT-5.2"},"databricks-gpt-5-4":{"cost":{"cache_read":0.25,"input":2.5,"output":15,"tiers":[{"cache_read":0.5,"input":5,"output":22.5,"tier":{"size":272000,"type":"context"}}]},"description":"Agent-ready GPT for coding and computer-use workflows at a lower cost","name":"GPT-5.4"},"databricks-gpt-5-4-mini":{"cost":{"cache_read":0.075,"input":0.75,"output":4.5},"description":"Strong small GPT for coding subagents, quick tool use, and high-volume work","name":"GPT-5.4 mini"},"databricks-gpt-5-4-nano":{"cost":{"cache_read":0.02,"input":0.2,"output":1.25},"description":"Cheapest GPT-5.4 lane for simple routing, extraction, and bulk automation","name":"GPT-5.4 nano"},"databricks-gpt-5-5":{"cost":{"cache_read":0.5,"input":5,"output":30,"tiers":[{"cache_read":1,"input":10,"output":45,"tier":{"size":272000,"type":"context"}}]},"description":"Default frontier GPT for coding, computer use, research, and knowledge work","name":"GPT-5.5"},"databricks-gpt-5-6-luna":{"cost":{"cache_read":0.1,"input":1,"output":6,"tiers":[{"cache_read":0.2,"input":2,"output":9,"tier":{"size":272000,"type":"context"}}]},"description":"Cost-efficient GPT-5.6 model for fast, high-volume workloads","name":"GPT-5.6 Luna"},"databricks-gpt-5-6-sol":{"cost":{"cache_read":0.5,"input":5,"output":30,"tiers":[{"cache_read":1,"input":10,"output":45,"tier":{"size":272000,"type":"context"}}]},"description":"Frontier GPT-5.6 model for complex professional work, coding, and agentic workflows","name":"GPT-5.6 Sol"},"databricks-gpt-5-6-terra":{"cost":{"cache_read":0.25,"input":2.5,"output":15,"tiers":[{"cache_read":0.5,"input":5,"output":22.5,"tier":{"size":272000,"type":"context"}}]},"description":"Balanced GPT-5.6 model for capable, cost-efficient everyday work","name":"GPT-5.6 Terra"},"databricks-gpt-5-mini":{"cost":{"cache_read":0.025,"input":0.25,"output":2},"description":"Small GPT-5 for responsive agents, coding help, and everyday automation","name":"GPT-5 Mini"},"databricks-gpt-5-nano":{"cost":{"cache_read":0.005,"input":0.05,"output":0.4},"description":"Tiny GPT-5 lane for routing, extraction, classification, and bulk jobs","name":"GPT-5 Nano"},"databricks-gpt-oss-120b":{"cost":{"input":0.072,"output":0.28},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"GPT OSS 120B"},"databricks-gpt-oss-20b":{"cost":{"input":0.05,"output":0.2},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"GPT OSS 20B"},"databricks-kimi-k2-7-code":{"cost":{"cache_read":0.19,"input":0.95,"output":4.0},"description":"Coding-focused Kimi model, stronger on long-horizon repo work with less overthinking","name":"Kimi K2.7 Code"}}},"deepinfra":{"models":{"MiniMaxAI/MiniMax-M2.5":{"cost":{"cache_read":0.03,"input":0.15,"output":1.15},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"MiniMax M2.5"},"MiniMaxAI/MiniMax-M2.7":{"cost":{"cache_read":0.05,"input":0.25,"output":1},"description":"Open MiniMax flagship for coding agents, office automation, and complex environments","name":"MiniMax-M2.7"},"MiniMaxAI/MiniMax-M3":{"cost":{"cache_read":0.056,"input":0.28,"output":1.1},"description":"MiniMax multimodal model for long-context coding, perception, and agent planning","name":"MiniMax-M3"},"Qwen/Qwen3-235B-A22B-Instruct-2507":{"cost":{"input":0.09,"output":0.55},"description":"Updated large open Qwen3 MoE instruct model for multilingual chat, coding, and tool use","name":"Qwen3 235B-A22B Instruct 2507"},"Qwen/Qwen3-32B":{"cost":{"input":0.08,"output":0.28},"description":"Dense open Qwen model for self-hosted chat, reasoning, and coding","name":"Qwen3 32B"},"Qwen/Qwen3-Coder-480B-A35B-Instruct-Turbo":{"cost":{"cache_read":0.1,"input":0.3,"output":1},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen3 Coder 480B A35B Instruct Turbo"},"Qwen/Qwen3-Max":{"cost":{"cache_read":0.24,"input":1.2,"output":6,"tiers":[{"cache_read":0.48,"input":2.4,"output":12,"tier":{"size":32000,"type":"context"}},{"cache_read":0.6,"input":3,"output":15,"tier":{"size":128000,"type":"context"}}]},"description":"Flagship Qwen3 model for coding agents, complex reasoning, and tool use","name":"Qwen3 Max"},"Qwen/Qwen3-Next-80B-A3B-Instruct":{"cost":{"input":0.09,"output":1.1},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3-Next 80B-A3B Instruct"},"Qwen/Qwen3.5-122B-A10B":{"cost":{"input":0.29,"output":2.4},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 122B-A10B"},"Qwen/Qwen3.5-27B":{"cost":{"input":0.26,"output":2.6},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 27B"},"Qwen/Qwen3.5-35B-A3B":{"cost":{"cache_read":0.05,"input":0.14,"output":1},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen 3.5 35B A3B"},"Qwen/Qwen3.5-397B-A17B":{"cost":{"cache_read":0.22,"input":0.45,"output":3},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen 3.5 397B A17B"},"Qwen/Qwen3.5-9B":{"cost":{"input":0.1,"output":0.15},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3.5 9B"},"Qwen/Qwen3.6-27B":{"cost":{"input":0.32,"output":3.2},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.6 27B"},"Qwen/Qwen3.6-35B-A3B":{"cost":{"input":0.1,"output":0.95},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.6 35B A3B"},"Qwen/Qwen3.7-Max":{"cost":{"cache_read":0.5,"input":2.5,"output":7.5,"tiers":[{"cache_read":1,"input":5,"output":15,"tier":{"size":32000,"type":"context"}},{"cache_read":1.25,"input":6.25,"output":18.5,"tier":{"size":128000,"type":"context"}}]},"description":"Qwen frontier model tuned for agent frameworks, coding assistants, and long tasks","name":"Qwen3.7 Max"},"Qwen/Qwen3.8-Max":{"cost":{"cache_read":0.206,"input":1.65,"output":4.951},"description":"2.4-trillion-parameter MoE flagship for coding, professional work, multimodal understanding, and long-horizon agentic workflows","name":"Qwen3.8 Max"},"XiaomiMiMo/MiMo-V2.5":{"cost":{"cache_read":0.08,"input":0.4,"output":2},"description":"Open MiMo model for multimodal coding agents and long-context automation","name":"MiMo-V2.5"},"XiaomiMiMo/MiMo-V2.5-Pro":{"cost":{"cache_read":0.2,"input":1,"output":3},"description":"Stronger MiMo Pro tier for multimodal reasoning and coding-agent execution","name":"MiMo-V2.5-Pro"},"deepseek-ai/DeepSeek-R1-0528":{"cost":{"cache_read":0.35,"input":0.5,"output":2.15},"description":"DeepSeek reasoning model for multi-step analysis, math, coding, and tools","name":"DeepSeek-R1-0528"},"deepseek-ai/DeepSeek-V3":{"cost":{"input":0.32,"output":0.89},"description":"Open DeepSeek MoE chat model for coding, math, and general reasoning","name":"DeepSeek-V3"},"deepseek-ai/DeepSeek-V3.1":{"cost":{"cache_read":0.13,"input":0.25,"output":0.95},"description":"Hybrid-reasoning DeepSeek model with thinking and non-thinking modes","name":"DeepSeek-V3.1"},"deepseek-ai/DeepSeek-V3.2":{"cost":{"cache_read":0.13,"input":0.26,"output":0.38},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek-V3.2"},"deepseek-ai/DeepSeek-V4-Flash":{"cost":{"cache_read":0.018,"input":0.09,"output":0.18},"description":"Fast DeepSeek V4 lane for economical reasoning, coding, and long-context work","name":"DeepSeek V4 Flash"},"deepseek-ai/DeepSeek-V4-Flash-0731":{"cost":{"cache_read":0.018,"input":0.09,"output":0.18},"description":"Official DeepSeek V4 Flash release with enhanced agentic capabilities and integrated DSpark speculative decoding","name":"DeepSeek V4 Flash 0731"},"deepseek-ai/DeepSeek-V4-Pro":{"cost":{"cache_read":0.1,"input":1.3,"output":2.6},"description":"Open MoE flagship with million-token context for coding and long agent runs","name":"DeepSeek V4 Pro"},"google/gemma-4-26B-A4B-it":{"cost":{"input":0.07,"output":0.34},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 4 26B A4B IT"},"google/gemma-4-31B-it":{"cost":{"input":0.13,"output":0.38},"description":"Largest Gemma 4 instruction model for open, self-hosted chat and reasoning","name":"Gemma 4 31B IT"},"google/gemma-4-E4B-it":{"cost":{"input":0.02,"output":0.1},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 4 E4B IT"},"meta-llama/Llama-3.3-70B-Instruct-Turbo":{"cost":{"input":0.1,"output":0.32},"description":"Compact Llama instruction model for fast chat and local deployment","name":"Llama 3.3 70B Turbo"},"meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8":{"cost":{"input":0.2,"output":0.8},"description":"Open multimodal Llama model for strong reasoning and fast responses","name":"Llama 4 Maverick 17B FP8"},"meta-llama/Llama-4-Scout-17B-16E-Instruct":{"cost":{"input":0.1,"output":0.3},"description":"Open multimodal Llama model for long-context analysis and efficient agents","name":"Llama 4 Scout 17B"},"moonshotai/Kimi-K2.5":{"cost":{"cache_read":0.07,"input":0.45,"output":2.25},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi K2.5"},"moonshotai/Kimi-K2.6":{"cost":{"cache_read":0.15,"input":0.75,"output":3.5},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi K2.6"},"moonshotai/Kimi-K2.7-Code":{"cost":{"cache_read":0.136,"input":0.68,"output":3.4},"description":"Coding-focused Kimi model, stronger on long-horizon repo work with less overthinking","name":"Kimi K2.7 Code"},"moonshotai/Kimi-K3":{"cost":{"cache_read":0.285,"input":2.85,"output":14.25},"description":"Multimodal Kimi model with 1M context and toggleable max-effort thinking for long-horizon agent work","name":"Kimi K3"},"nvidia/Llama-3.3-Nemotron-Super-49B-v1.5":{"cost":{"input":0.4,"output":0.4},"description":"Nemotron model for efficient reasoning, coding, and specialized AI agents","name":"Llama 3.3 Nemotron Super 49B v1.5"},"nvidia/Nemotron-3-Nano-30B-A3B":{"cost":{"cache_read":0.025,"input":0.05,"output":0.2},"description":"Small Nemotron 3 MoE for efficient coding, math, and long-context agents","name":"Nemotron 3 Nano 30B A3B"},"nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning":{"cost":{"input":0.2,"output":0.8},"description":"Open Nemotron omni model combining reasoning with text, vision, and audio","name":"Nemotron 3 Nano Omni 30B A3B Reasoning"},"openai/gpt-oss-120b":{"cost":{"input":0.037,"output":0.17},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"GPT OSS 120B"},"openai/gpt-oss-20b":{"cost":{"input":0.03,"output":0.14},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"GPT OSS 20B"},"stepfun-ai/Step-3.7-Flash":{"cost":{"cache_read":0.04,"input":0.2,"output":1.15},"description":"Newer StepFun flash model for faster agents, coding, and multimodal prompts","name":"Step 3.7 Flash"},"tencent/Hy3":{"cost":{"cache_read":0.035,"input":0.14,"output":0.58},"description":"Tencent Hy reasoning model for coding, instruction following, and agent tasks","name":"Hy3"},"thinkingmachines/Inkling":{"cost":{"cache_read":0.16,"input":0.95,"output":4.05},"description":"Multimodal MoE reasoning model (975B total, 41B active) for text, image, and audio","name":"Inkling"},"thinkingmachines/Inkling-Small":{"cost":{"cache_read":0.1,"input":0.45,"output":1.2},"description":"Multimodal MoE reasoning model (276B total, 12B active) for text, image, and audio","name":"Inkling Small"},"zai-org/GLM-4.6":{"cost":{"cache_read":0.1,"input":0.5,"output":2},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM-4.6"},"zai-org/GLM-4.7":{"cost":{"cache_read":0.08,"input":0.4,"output":1.75},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM-4.7"},"zai-org/GLM-4.7-Flash":{"cost":{"cache_read":0.01,"input":0.06,"output":0.4},"description":"Efficient GLM model for fast reasoning, coding, and agent workflows","name":"GLM-4.7-Flash"},"zai-org/GLM-5":{"cost":{"cache_read":0.12,"input":0.6,"output":2.08},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM-5"},"zai-org/GLM-5.1":{"cost":{"cache_read":0.205,"input":1.05,"output":3.5},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM-5.1"},"zai-org/GLM-5.2":{"cost":{"cache_read":0.14,"input":0.75,"output":2.4},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM-5.2"}}},"deepseek":{"models":{"deepseek-chat":{"cost":{"cache_read":0.0028,"input":0.14,"output":0.28},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek Chat"},"deepseek-reasoner":{"cost":{"cache_read":0.0028,"input":0.14,"output":0.28,"reasoning":0.28},"description":"DeepSeek reasoning model for multi-step analysis, math, coding, and tools","name":"DeepSeek Reasoner"},"deepseek-v4-flash":{"cost":{"cache_read":0.0028,"input":0.14,"output":0.28,"reasoning":0.28},"description":"Official DeepSeek V4 Flash release with enhanced agentic capabilities and integrated DSpark speculative decoding","name":"DeepSeek V4 Flash"},"deepseek-v4-pro":{"cost":{"cache_read":0.003625,"input":0.435,"output":0.87,"reasoning":0.87},"description":"Open MoE flagship with million-token context for coding and long agent runs","name":"DeepSeek V4 Pro"}}},"gemini":{"models":{"deep-research-max-preview-04-2026":{"cost":{"cache_read":0.2,"input":2,"output":12,"tiers":[{"cache_read":0.4,"input":4,"output":18,"tier":{"size":200000,"type":"context"}}]},"description":"Maximum-comprehensiveness agentic researcher for multi-step investigation, synthesis, and cited reports","name":"Deep Research Max Preview (Apr-21-2026)"},"deep-research-preview-04-2026":{"cost":{"cache_read":0.2,"input":2,"output":12,"tiers":[{"cache_read":0.4,"input":4,"output":18,"tier":{"size":200000,"type":"context"}}]},"description":"Agentic model for autonomous multi-step research, synthesis, and cited reports","name":"Deep Research Preview (Apr-21-2026)"},"gemini-2.0-flash":{"cost":{"cache_read":0.025,"input":0.1,"output":0.4},"description":"Earlier Gemini Flash workhorse for responsive multimodal apps and tool use","name":"Gemini 2.0 Flash"},"gemini-2.0-flash-lite":{"cost":{"input":0.075,"output":0.3},"description":"Legacy model retained for compatibility with older integrations","name":"Gemini 2.0 Flash-Lite"},"gemini-2.5-computer-use-preview-10-2025":{"cost":{"input":1.25,"output":10,"tiers":[{"input":2.5,"output":15,"tier":{"size":200000,"type":"context"}}]},"description":"Specialized Gemini 2.5 model for browser-control agents that automate UI tasks","name":"Gemini 2.5 Computer Use Preview 10-2025"},"gemini-2.5-flash":{"cost":{"cache_read":0.03,"input":0.3,"input_audio":1.0,"output":2.5},"description":"Fast Gemini workhorse for multimodal apps where latency and price matter","name":"Gemini 2.5 Flash"},"gemini-2.5-flash-image":{"cost":{"cache_read":0.075,"input":0.3,"output":30},"description":"Nano Banana image model for fast generation, edits, and character-consistent assets","name":"Nano Banana"},"gemini-2.5-flash-lite":{"cost":{"cache_read":0.01,"input":0.1,"input_audio":0.3,"output":0.4},"description":"Lean Gemini 2.5 lane for cheap multimodal traffic and quick agents","name":"Gemini 2.5 Flash-Lite"},"gemini-2.5-flash-preview-tts":{"cost":{"input":0.5,"output":10},"description":"Speech generation model for controllable voice, narration, and audio delivery","name":"Gemini 2.5 Flash Preview TTS"},"gemini-2.5-pro":{"cost":{"cache_read":0.125,"input":1.25,"output":10.0,"tiers":[{"cache_read":0.25,"input":2.5,"output":15.0,"tier":{"size":200000}}]},"description":"Google's proven reasoning model for coding, math, and multimodal analysis","name":"Gemini 2.5 Pro"},"gemini-2.5-pro-preview-tts":{"cost":{"input":1,"output":20},"description":"Speech generation model for controllable voice, narration, and audio delivery","name":"Gemini 2.5 Pro Preview TTS"},"gemini-3-flash-preview":{"cost":{"cache_read":0.05,"input":0.5,"input_audio":1.0,"output":3.0},"description":"New Gemini flash lane bringing frontier-style multimodal reasoning to cheaper runs","name":"Gemini 3 Flash Preview"},"gemini-3-pro-image":{"cost":{"input":2,"output":120},"description":"Nano Banana Pro for higher-fidelity image generation and design-heavy edits","name":"Nano Banana Pro"},"gemini-3-pro-image-preview":{"cost":{"input":2,"output":120},"description":"Nano Banana Pro for higher-fidelity image generation and design-heavy edits","name":"Nano Banana Pro"},"gemini-3-pro-preview":{"cost":{"cache_read":0.2,"input":2,"output":12,"tiers":[{"cache_read":0.4,"input":4,"output":18,"tier":{"size":200000}}]},"description":"Preview Gemini flagship for complex reasoning, coding, and rich multimodal prompts","name":"Gemini 3 Pro Preview"},"gemini-3.1-flash-image":{"cost":{"input":0.5,"output":60},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Nano Banana 2"},"gemini-3.1-flash-image-preview":{"cost":{"input":0.5,"output":60},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Nano Banana 2"},"gemini-3.1-flash-lite":{"cost":{"cache_read":0.025,"input":0.25,"input_audio":0.5,"output":1.5},"description":"Low-latency Gemini model for high-volume multimodal and agent workloads","name":"Gemini 3.1 Flash Lite"},"gemini-3.1-flash-lite-image":{"cost":{"input":0.25,"output":30},"description":"Fastest, most cost-efficient Gemini image model for high-volume 1K generation and editing","name":"Nano Banana 2 Lite"},"gemini-3.1-flash-lite-preview":{"cost":{"cache_read":0.025,"input":0.25,"input_audio":0.5,"output":1.5},"description":"Legacy model retained for compatibility with older integrations","name":"Gemini 3.1 Flash Lite Preview"},"gemini-3.1-flash-live-preview":{"cost":{"input":0.75,"input_audio":3.0,"output":4.5,"output_audio":12.0},"description":"High-quality, low-latency Live API model for real-time dialogue and voice-first AI applications","name":"Gemini 3.1 Flash Live Preview"},"gemini-3.1-flash-tts-preview":{"cost":{"input":1,"output":20},"description":"Low-latency speech generation with steerable prompts and expressive audio tags","name":"Gemini 3.1 Flash TTS Preview"},"gemini-3.1-pro-preview":{"cost":{"cache_read":0.2,"input":2.0,"output":12.0,"tiers":[{"cache_read":0.4,"input":4.0,"output":18.0,"tier":{"size":200000}}]},"description":"Reasoning-first Gemini preview for agentic coding and complex problem solving","name":"Gemini 3.1 Pro Preview"},"gemini-3.1-pro-preview-customtools":{"cost":{"cache_read":0.2,"input":2.0,"output":12.0,"tiers":[{"cache_read":0.4,"input":4.0,"output":18.0,"tier":{"size":200000}}]},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Gemini 3.1 Pro Preview Custom Tools"},"gemini-3.5-flash":{"cost":{"cache_read":0.15,"input":1.5,"input_audio":1.5,"output":9.0},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini 3.5 Flash"},"gemini-3.5-flash-lite":{"cost":{"cache_read":0.03,"input":0.3,"output":2.5},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini 3.5 Flash Lite"},"gemini-3.5-live-translate-preview":{"cost":{"input":3.5,"input_audio":3.5,"output":21,"output_audio":21},"description":"Low-latency audio-to-audio model for real-time speech translation across 70+ languages","name":"Gemini 3.5 Live Translate Preview"},"gemini-3.6-flash":{"cost":{"cache_read":0.15,"input":1.5,"input_audio":1.5,"output":7.5},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini 3.6 Flash"},"gemini-embedding-001":{"cost":{"input":0.15,"output":0},"description":"Embedding model for semantic search, retrieval, clustering, and ranking pipelines","name":"Gemini Embedding 001"},"gemini-embedding-2":{"cost":{"input":0.2,"input_audio":6.5,"output":0},"description":"Multimodal embedding model mapping text, images, video, audio, and PDFs into a unified embedding space","name":"Gemini Embedding 2"},"gemini-flash-latest":{"cost":{"cache_read":0.15,"input":1.5,"input_audio":1.5,"output":9.0},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini Flash Latest"},"gemini-flash-lite-latest":{"cost":{"cache_read":0.025,"input":0.25,"input_audio":0.5,"output":1.5},"description":"Low-latency Gemini model for high-volume multimodal and agent workloads","name":"Gemini Flash-Lite Latest"},"gemini-omni-flash-preview":{"cost":{"input":1.5,"output":17.5},"description":"Video generation and editing model for fast, conversational text- and image-to-video workflows","name":"Gemini Omni Flash Preview"},"gemini-robotics-er-1.6-preview":{"cost":{"input":1.0,"input_audio":2.0,"output":5.0},"description":"Vision-language model for embodied reasoning: spatial understanding, task planning, and physical-world agentic robotics","name":"Gemini Robotics-ER 1.6 Preview"},"gemma-4-26b-a4b-it":{"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 4 26B A4B IT"},"gemma-4-31b-it":{"description":"Largest Gemma 4 instruction model for open, self-hosted chat and reasoning","name":"Gemma 4 31B IT"},"lyria-3-clip-preview":{"cost":{"input":0,"output":0},"description":"Music generation model for short 30-second clips, loops, and previews from text or image prompts","name":"Lyria 3 Clip Preview"},"lyria-3-pro-preview":{"cost":{"input":0,"output":0},"description":"Music generation model for full-length songs from text or images with vocals and structure","name":"Lyria 3 Pro Preview"},"veo-3.1-fast-generate-preview":{"description":"Video model for prompt-guided generation, editing, and motion workflows","name":"Veo 3.1 fast"},"veo-3.1-generate-preview":{"description":"Video model for prompt-guided generation, editing, and motion workflows","name":"Veo 3.1"},"veo-3.1-lite-generate-preview":{"description":"Video model for prompt-guided generation, editing, and motion workflows","name":"Veo 3.1 lite"}}},"github_copilot":{"models":{"claude-fable-5":{"cost":{"cache_read":1,"cache_write":12.5,"input":10,"output":50},"description":"Claude model for creative writing, analysis, and controlled agent workflows","name":"Claude Fable 5"},"claude-haiku-4.5":{"cost":{"cache_read":0.1,"cache_write":1.25,"input":1,"output":5},"description":"Fast Claude lane for lightweight agents, office tasks, and responsive chat","name":"Claude Haiku 4.5 (latest)"},"claude-opus-4.5":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4.5 (latest)"},"claude-opus-4.6":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"High-end Claude for difficult coding, planning, and slower expert reasoning","name":"Claude Opus 4.6"},"claude-opus-4.7":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Stronger Opus tier for advanced software work and high-stakes reasoning","name":"Claude Opus 4.7"},"claude-opus-4.8":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Top Claude Opus tier for the hardest reasoning, coding, and long-horizon agents","name":"Claude Opus 4.8"},"claude-opus-5":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Strongest Claude Opus model for coding, agents, and professional work","name":"Claude Opus 5"},"claude-sonnet-4":{"cost":{"cache_read":0.3,"cache_write":3.75,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude Sonnet 4 (latest)"},"claude-sonnet-4.5":{"cost":{"cache_read":0.3,"cache_write":3.75,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude Sonnet 4.5 (latest)"},"claude-sonnet-4.6":{"cost":{"cache_read":0.3,"cache_write":3.75,"input":3,"output":15},"description":"Claude workhorse for coding agents, careful analysis, and production cost control","name":"Claude Sonnet 4.6"},"claude-sonnet-5":{"cost":{"cache_read":0.2,"cache_write":2.5,"input":2,"output":10},"description":"Everyday Claude agent model for coding, planning, browsing, and general work","name":"Claude Sonnet 5"},"gemini-3.1-pro-preview":{"cost":{"cache_read":0.2,"input":2,"output":12,"tiers":[{"cache_read":0.4,"input":4,"output":18,"tier":{"size":200000,"type":"context"}}]},"description":"Reasoning-first Gemini preview for agentic coding and complex problem solving","name":"Gemini 3.1 Pro Preview"},"gemini-3.5-flash":{"cost":{"cache_read":0.15,"input":1.5,"input_audio":1.5,"output":9},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini 3.5 Flash"},"gemini-3.6-flash":{"cost":{"cache_read":0.15,"input":1.5,"output":7.5},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini 3.6 Flash"},"gpt-4.1":{"cost":{"cache_read":0.5,"input":2,"output":8},"description":"Long-lived GPT workhorse for coding, instruction following, and production apps","name":"GPT-4.1"},"gpt-5-mini":{"cost":{"cache_read":0.025,"input":0.25,"output":2},"description":"Small GPT-5 for responsive agents, coding help, and everyday automation","name":"GPT-5 Mini"},"gpt-5.2":{"cost":{"cache_read":0.175,"input":1.75,"output":14},"description":"Reliable GPT generation for broad coding, writing, and tool-assisted product work","name":"GPT-5.2"},"gpt-5.2-codex":{"cost":{"cache_read":0.175,"input":1.75,"output":14},"description":"Code-specialist GPT for repository edits, reviews, and long-running software agents","name":"GPT-5.2 Codex"},"gpt-5.3-codex":{"cost":{"cache_read":0.175,"input":1.75,"output":14},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.3 Codex"},"gpt-5.4":{"cost":{"cache_read":0.25,"input":2.5,"output":15,"tiers":[{"cache_read":0.5,"input":5,"output":22.5,"tier":{"size":272000,"type":"context"}}]},"description":"Agent-ready GPT for coding and computer-use workflows at a lower cost","name":"GPT-5.4"},"gpt-5.4-mini":{"cost":{"cache_read":0.075,"input":0.75,"output":4.5},"description":"Strong small GPT for coding subagents, quick tool use, and high-volume work","name":"GPT-5.4 mini"},"gpt-5.4-nano":{"cost":{"cache_read":0.02,"input":0.2,"output":1.25},"description":"Cheapest GPT-5.4 lane for simple routing, extraction, and bulk automation","name":"GPT-5.4 nano"},"gpt-5.5":{"cost":{"cache_read":0.5,"input":5,"output":30,"tiers":[{"cache_read":1,"input":10,"output":45,"tier":{"size":272000,"type":"context"}}]},"description":"Default frontier GPT for coding, computer use, research, and knowledge work","name":"GPT-5.5"},"gpt-5.6-luna":{"cost":{"cache_read":0.02,"input":0.2,"output":1.2,"tiers":[{"cache_read":0.04,"input":0.4,"output":1.8,"tier":{"size":200000,"type":"context"}}]},"description":"Cost-efficient GPT-5.6 model for fast, high-volume workloads","name":"GPT-5.6 Luna"},"gpt-5.6-sol":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":30,"tiers":[{"cache_read":1,"cache_write":12.5,"input":10,"output":45,"tier":{"size":272000,"type":"context"}}]},"description":"Frontier GPT-5.6 model for complex professional work, coding, and agentic workflows","name":"GPT-5.6 Sol"},"gpt-5.6-terra":{"cost":{"cache_read":0.2,"input":2.0,"output":12.0,"tiers":[{"cache_read":0.4,"input":4.0,"output":18.0,"tier":{"size":272000,"type":"context"}}]},"description":"Balanced GPT-5.6 model for capable, cost-efficient everyday work","name":"GPT-5.6 Terra"},"grok-4.5":{"cost":{"cache_read":0.5,"input":2,"output":6,"tiers":[{"cache_read":1,"input":4,"output":12,"tier":{"size":200000,"type":"context"}}]},"description":"xAI's latest Grok for chat, coding, agentic tools, and lower hallucination risk","name":"Grok 4.5"},"kimi-k2.7-code":{"cost":{"cache_read":0.19,"input":0.95,"output":4.0},"description":"Coding-focused Kimi model, stronger on long-horizon repo work with less overthinking","name":"Kimi K2.7 Code"},"kimi-k3":{"cost":{"cache_read":0.3,"input":3.0,"output":15.0},"description":"Multimodal Kimi model with 1M context and toggleable max-effort thinking for long-horizon agent work","name":"Kimi K3"},"mai-code-1-flash-picker":{"cost":{"cache_read":0.075,"input":0.75,"output":4.5},"description":"Microsoft coding model built for fast, efficient assistance in everyday developer workflows","name":"MAI-Code-1-Flash"}}},"groq":{"models":{"allam-2-7b":{"cost":{"input":0,"output":0},"description":"ALLaM-2-7b instruction tuned model by SDAIA","name":"ALLaM-2-7b"},"canopylabs/orpheus-arabic-saudi":{"description":"Speech generation model for controllable voice, narration, and audio delivery","name":"Canopy Labs Orpheus Arabic Saudi"},"canopylabs/orpheus-v1-english":{"description":"Speech generation model for controllable voice, narration, and audio delivery","name":"Canopy Labs Orpheus V1 English"},"groq/compound":{"description":"General-purpose chat model for instruction following, writing, and analysis","name":"Compound"},"groq/compound-mini":{"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Compound Mini"},"llama-3.1-8b-instant":{"cost":{"input":0.05,"output":0.08},"description":"Compact Llama instruction model for fast chat and local deployment","name":"Llama 3.1 8B"},"llama-3.3-70b-versatile":{"cost":{"input":0.59,"output":0.79},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.3 70B"},"meta-llama/llama-prompt-guard-2-22m":{"cost":{"input":0.03,"output":0.03},"description":"Safety model for policy screening, moderation, and risk-aware routing workflows","name":"Llama Prompt Guard 2 22M"},"meta-llama/llama-prompt-guard-2-86m":{"cost":{"input":0.04,"output":0.04},"description":"Safety model for policy screening, moderation, and risk-aware routing workflows","name":"Prompt Guard 2 86M"},"openai/gpt-oss-120b":{"cost":{"cache_read":0.075,"input":0.15,"output":0.6},"description":"Open GPT reasoning model for self-hosted agents and controllable deployments","name":"GPT OSS 120B"},"openai/gpt-oss-20b":{"cost":{"cache_read":0.0375,"input":0.075,"output":0.3},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"GPT OSS 20B"},"openai/gpt-oss-safeguard-20b":{"cost":{"input":0.075,"output":0.3},"description":"Safety model for policy screening, moderation, and risk-aware routing workflows","name":"Safety GPT OSS 20B"},"qwen/qwen3.6-27b":{"cost":{"cache_read":0.3,"input":0.6,"output":3.0},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.6 27B"},"whisper-large-v3":{"description":"Speech transcription model for accurate audio-to-text and captioning workflows","name":"Whisper"},"whisper-large-v3-turbo":{"description":"Speech transcription model for accurate audio-to-text and captioning workflows","name":"Whisper Large V3 Turbo"}}},"helicone":{"models":{"chatgpt-4o-latest":{"cost":{"cache_read":2.5,"input":5,"output":20},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"OpenAI ChatGPT-4o"},"claude-3-haiku-20240307":{"cost":{"cache_read":0.03,"cache_write":0.3,"input":0.25,"output":1.25},"description":"Fast Claude model for responsive assistance, classification, and lightweight agents","name":"Anthropic: Claude 3 Haiku"},"claude-3.5-haiku":{"cost":{"cache_read":0.08,"cache_write":1,"input":0.7999999999999999,"output":4},"description":"Fast Claude model for responsive assistance, classification, and lightweight agents","name":"Anthropic: Claude 3.5 Haiku"},"claude-3.5-sonnet-v2":{"cost":{"cache_read":0.30000000000000004,"cache_write":3.75,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Anthropic: Claude 3.5 Sonnet v2"},"claude-3.7-sonnet":{"cost":{"cache_read":0.30000000000000004,"cache_write":3.75,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Anthropic: Claude 3.7 Sonnet"},"claude-4.5-haiku":{"cost":{"cache_read":0.09999999999999999,"cache_write":1.25,"input":1,"output":5},"description":"Fast Claude model for responsive assistance, classification, and lightweight agents","name":"Anthropic: Claude 4.5 Haiku"},"claude-4.5-opus":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Anthropic: Claude Opus 4.5"},"claude-4.5-sonnet":{"cost":{"cache_read":0.30000000000000004,"cache_write":3.75,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Anthropic: Claude Sonnet 4.5"},"claude-haiku-4-5-20251001":{"cost":{"cache_read":0.09999999999999999,"cache_write":1.25,"input":1,"output":5},"description":"Fast Claude model for responsive assistance, classification, and lightweight agents","name":"Anthropic: Claude 4.5 Haiku (20251001)"},"claude-opus-4":{"cost":{"cache_read":1.5,"cache_write":18.75,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Anthropic: Claude Opus 4"},"claude-opus-4-1":{"cost":{"cache_read":1.5,"cache_write":18.75,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Anthropic: Claude Opus 4.1"},"claude-opus-4-1-20250805":{"cost":{"cache_read":1.5,"cache_write":18.75,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Anthropic: Claude Opus 4.1 (20250805)"},"claude-sonnet-4":{"cost":{"cache_read":0.30000000000000004,"cache_write":3.75,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Anthropic: Claude Sonnet 4"},"claude-sonnet-4-5-20250929":{"cost":{"cache_read":0.30000000000000004,"cache_write":3.75,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Anthropic: Claude Sonnet 4.5 (20250929)"},"deepseek-r1-distill-llama-70b":{"cost":{"input":0.03,"output":0.13},"description":"DeepSeek reasoning model for multi-step analysis, math, coding, and tools","name":"DeepSeek R1 Distill Llama 70B"},"deepseek-reasoner":{"cost":{"cache_read":0.07,"input":0.56,"output":1.68},"description":"DeepSeek reasoning model for multi-step analysis, math, coding, and tools","name":"DeepSeek Reasoner"},"deepseek-tng-r1t2-chimera":{"cost":{"input":0.3,"output":1.2},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek TNG R1T2 Chimera"},"deepseek-v3":{"cost":{"cache_read":0.07,"input":0.56,"output":1.68},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek V3"},"deepseek-v3.1-terminus":{"cost":{"cache_read":0.21600000000000003,"input":0.27,"output":1},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek V3.1 Terminus"},"deepseek-v3.2":{"cost":{"input":0.27,"output":0.41},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek V3.2"},"ernie-4.5-21b-a3b-thinking":{"cost":{"input":0.07,"output":0.28},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"Baidu Ernie 4.5 21B A3B Thinking"},"gemini-2.5-flash":{"cost":{"cache_read":0.075,"cache_write":0.3,"input":0.3,"output":2.5},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Google Gemini 2.5 Flash"},"gemini-2.5-flash-lite":{"cost":{"cache_read":0.024999999999999998,"cache_write":0.09999999999999999,"input":0.09999999999999999,"output":0.39999999999999997},"description":"Low-latency Gemini model for high-volume multimodal and agent workloads","name":"Google Gemini 2.5 Flash Lite"},"gemini-2.5-pro":{"cost":{"cache_read":0.3125,"cache_write":1.25,"input":1.25,"output":10},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Google Gemini 2.5 Pro"},"gemini-3-pro-preview":{"cost":{"cache_read":0.19999999999999998,"input":2,"output":12},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Google Gemini 3 Pro Preview"},"gemma-3-12b-it":{"cost":{"input":0.049999999999999996,"output":0.09999999999999999},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Google Gemma 3 12B"},"gemma2-9b-it":{"cost":{"input":0.01,"output":0.03},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Google Gemma 2"},"glm-4.6":{"cost":{"input":0.44999999999999996,"output":1.5},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"Zai GLM-4.6"},"gpt-4.1":{"cost":{"cache_read":0.5,"input":2,"output":8},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"OpenAI GPT-4.1"},"gpt-4.1-mini":{"cost":{"cache_read":0.09999999999999999,"input":0.39999999999999997,"output":1.5999999999999999},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"OpenAI GPT-4.1 Mini"},"gpt-4.1-mini-2025-04-14":{"cost":{"cache_read":0.09999999999999999,"input":0.39999999999999997,"output":1.5999999999999999},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"OpenAI GPT-4.1 Mini"},"gpt-4.1-nano":{"cost":{"cache_read":0.024999999999999998,"input":0.09999999999999999,"output":0.39999999999999997},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"OpenAI GPT-4.1 Nano"},"gpt-4o":{"cost":{"cache_read":1.25,"input":2.5,"output":10},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"OpenAI GPT-4o"},"gpt-4o-mini":{"cost":{"cache_read":0.075,"input":0.15,"output":0.6},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"OpenAI GPT-4o-mini"},"gpt-5":{"cost":{"cache_read":0.12500000000000003,"input":1.25,"output":10},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"OpenAI GPT-5"},"gpt-5-chat-latest":{"cost":{"cache_read":0.12500000000000003,"input":1.25,"output":10},"description":"Chat-tuned GPT model for conversational assistance, writing, and tool workflows","name":"OpenAI GPT-5 Chat Latest"},"gpt-5-codex":{"cost":{"cache_read":0.12500000000000003,"input":1.25,"output":10},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"OpenAI: GPT-5 Codex"},"gpt-5-mini":{"cost":{"cache_read":0.024999999999999998,"input":0.25,"output":2},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"OpenAI GPT-5 Mini"},"gpt-5-nano":{"cost":{"cache_read":0.005,"input":0.049999999999999996,"output":0.39999999999999997},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"OpenAI GPT-5 Nano"},"gpt-5-pro":{"cost":{"input":15,"output":120},"description":"Frontier GPT model for professional reasoning, coding, and multimodal work","name":"OpenAI: GPT-5 Pro"},"gpt-5.1":{"cost":{"cache_read":0.12500000000000003,"input":1.25,"output":10},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"OpenAI GPT-5.1"},"gpt-5.1-chat-latest":{"cost":{"cache_read":0.12500000000000003,"input":1.25,"output":10},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"OpenAI GPT-5.1 Chat"},"gpt-5.1-codex":{"cost":{"cache_read":0.12500000000000003,"input":1.25,"output":10},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"OpenAI: GPT-5.1 Codex"},"gpt-5.1-codex-mini":{"cost":{"cache_read":0.024999999999999998,"input":0.25,"output":2},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"OpenAI: GPT-5.1 Codex Mini"},"gpt-oss-120b":{"cost":{"input":0.04,"output":0.16},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"OpenAI GPT-OSS 120b"},"gpt-oss-20b":{"cost":{"input":0.049999999999999996,"output":0.19999999999999998},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"OpenAI GPT-OSS 20b"},"grok-3":{"cost":{"cache_read":0.75,"input":3,"output":15},"description":"Grok model for agentic tool use, reasoning, coding, and live assistance","name":"xAI Grok 3"},"grok-3-mini":{"cost":{"cache_read":0.075,"input":0.3,"output":0.5},"description":"Fast Grok model for responsive chat, reasoning, and tool-assisted work","name":"xAI Grok 3 Mini"},"grok-4":{"cost":{"cache_read":0.75,"input":3,"output":15},"description":"Grok model for agentic tool use, reasoning, coding, and live assistance","name":"xAI Grok 4"},"grok-4-1-fast-non-reasoning":{"cost":{"cache_read":0.049999999999999996,"input":0.19999999999999998,"output":0.5},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"xAI Grok 4.1 Fast Non-Reasoning"},"grok-4-1-fast-reasoning":{"cost":{"cache_read":0.049999999999999996,"input":0.19999999999999998,"output":0.5},"description":"Fast Grok model for responsive chat, reasoning, and tool-assisted work","name":"xAI Grok 4.1 Fast Reasoning"},"grok-4-fast-non-reasoning":{"cost":{"cache_read":0.049999999999999996,"input":0.19999999999999998,"output":0.5},"description":"Fast Grok model for responsive chat, reasoning, and tool-assisted work","name":"xAI Grok 4 Fast Non-Reasoning"},"grok-4-fast-reasoning":{"cost":{"cache_read":0.049999999999999996,"input":0.19999999999999998,"output":0.5},"description":"Fast Grok model for responsive chat, reasoning, and tool-assisted work","name":"xAI: Grok 4 Fast Reasoning"},"grok-code-fast-1":{"cost":{"cache_read":0.02,"input":0.19999999999999998,"output":1.5},"description":"Fast Grok model for responsive chat, reasoning, and tool-assisted work","name":"xAI Grok Code Fast 1"},"hermes-2-pro-llama-3-8b":{"cost":{"input":0.14,"output":0.14},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Hermes 2 Pro Llama 3 8B"},"kimi-k2-0711":{"cost":{"input":0.5700000000000001,"output":2.3},"description":"Kimi model for long-context chat, coding, and agentic reasoning","name":"Kimi K2 (07/11)"},"kimi-k2-0905":{"cost":{"cache_read":0.39999999999999997,"input":0.5,"output":2},"description":"Kimi model for long-context chat, coding, and agentic reasoning","name":"Kimi K2 (09/05)"},"kimi-k2-thinking":{"cost":{"input":0.48,"output":2},"description":"Kimi reasoning model for long-horizon research, planning, and tool use","name":"Kimi K2 Thinking"},"llama-3.1-8b-instant":{"cost":{"input":0.049999999999999996,"output":0.08},"description":"Compact Llama instruction model for fast chat and local deployment","name":"Meta Llama 3.1 8B Instant"},"llama-3.1-8b-instruct":{"cost":{"input":0.02,"output":0.049999999999999996},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Meta Llama 3.1 8B Instruct"},"llama-3.1-8b-instruct-turbo":{"cost":{"input":0.02,"output":0.03},"description":"Compact Llama instruction model for fast chat and local deployment","name":"Meta Llama 3.1 8B Instruct Turbo"},"llama-3.3-70b-instruct":{"cost":{"input":0.13,"output":0.39},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Meta Llama 3.3 70B Instruct"},"llama-3.3-70b-versatile":{"cost":{"input":0.59,"output":0.7899999999999999},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Meta Llama 3.3 70B Versatile"},"llama-4-maverick":{"cost":{"input":0.15,"output":0.6},"description":"Open multimodal Llama model for strong reasoning and fast responses","name":"Meta Llama 4 Maverick 17B 128E"},"llama-4-scout":{"cost":{"input":0.08,"output":0.3},"description":"Open multimodal Llama model for long-context analysis and efficient agents","name":"Meta Llama 4 Scout 17B 16E"},"llama-guard-4":{"cost":{"input":0.21,"output":0.21},"description":"Safety model for policy screening, moderation, and risk-aware routing workflows","name":"Meta Llama Guard 4 12B"},"llama-prompt-guard-2-22m":{"cost":{"input":0.01,"output":0.01},"description":"Safety model for policy screening, moderation, and risk-aware routing workflows","name":"Meta Llama Prompt Guard 2 22M"},"llama-prompt-guard-2-86m":{"cost":{"input":0.01,"output":0.01},"description":"Safety model for policy screening, moderation, and risk-aware routing workflows","name":"Meta Llama Prompt Guard 2 86M"},"mistral-large-2411":{"cost":{"input":2,"output":6},"description":"Flagship Mistral model for advanced reasoning, coding, and multilingual work","name":"Mistral-Large"},"mistral-nemo":{"cost":{"input":20,"output":40},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral Nemo"},"mistral-small":{"cost":{"input":0.075,"output":0.2},"description":"Efficient Mistral model for fast chat, extraction, and production assistants","name":"Mistral Small 3.2"},"o1":{"cost":{"cache_read":7.5,"input":15,"output":60},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"OpenAI: o1"},"o1-mini":{"cost":{"cache_read":0.55,"input":1.1,"output":4.4},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"OpenAI: o1-mini"},"o3":{"cost":{"cache_read":0.5,"input":2,"output":8},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"OpenAI o3"},"o3-mini":{"cost":{"cache_read":0.55,"input":1.1,"output":4.4},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"OpenAI o3 Mini"},"o3-pro":{"cost":{"input":20,"output":80},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"OpenAI o3 Pro"},"o4-mini":{"cost":{"cache_read":0.275,"input":1.1,"output":4.4},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"OpenAI o4 Mini"},"qwen2.5-coder-7b-fast":{"cost":{"input":0.03,"output":0.09},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen2.5 Coder 7B fast"},"qwen3-235b-a22b-thinking":{"cost":{"input":0.3,"output":2.9000000000000004},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3 235B A22B Thinking"},"qwen3-30b-a3b":{"cost":{"input":0.08,"output":0.29},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3 30B A3B"},"qwen3-32b":{"cost":{"input":0.29,"output":0.59},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3 32B"},"qwen3-coder":{"cost":{"input":0.22,"output":0.95},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen3 Coder 480B A35B Instruct Turbo"},"qwen3-coder-30b-a3b-instruct":{"cost":{"input":0.09999999999999999,"output":0.3},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen3 Coder 30B A3B Instruct"},"qwen3-next-80b-a3b-instruct":{"cost":{"input":0.14,"output":1.4},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3 Next 80B A3B Instruct"},"qwen3-vl-235b-a22b-instruct":{"cost":{"input":0.3,"output":1.5},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3 VL 235B A22B Instruct"},"sonar":{"cost":{"input":1,"output":1},"description":"Sonar search model for current answers, retrieval, and citation-backed chat","name":"Perplexity Sonar"},"sonar-deep-research":{"cost":{"input":2,"output":8},"description":"Sonar search model for current answers, retrieval, and citation-backed chat","name":"Perplexity Sonar Deep Research"},"sonar-pro":{"cost":{"input":3,"output":15},"description":"Advanced Sonar search model for deeper research and cited synthesis","name":"Perplexity Sonar Pro"},"sonar-reasoning":{"cost":{"input":1,"output":5},"description":"Web-grounded reasoning model for multi-step research and cited answers","name":"Perplexity Sonar Reasoning"},"sonar-reasoning-pro":{"cost":{"input":2,"output":8},"description":"Web-grounded reasoning model for multi-step research and cited answers","name":"Perplexity Sonar Reasoning Pro"}}},"huggingface":{"models":{"MiniMaxAI/MiniMax-M2":{"cost":{"input":0.3,"output":1.2},"description":"Efficient open MiniMax model built for coding agents and tool-heavy workflows","name":"MiniMax-M2"},"MiniMaxAI/MiniMax-M2.1":{"cost":{"input":0.3,"output":1.2},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"MiniMax-M2.1"},"MiniMaxAI/MiniMax-M2.5":{"cost":{"cache_read":0.03,"input":0.3,"output":1.2},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"MiniMax-M2.5"},"MiniMaxAI/MiniMax-M2.7":{"cost":{"cache_read":0.06,"input":0.3,"output":1.2},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"MiniMax-M2.7"},"MiniMaxAI/MiniMax-M3":{"cost":{"input":0.3,"output":1.2},"description":"MiniMax multimodal model for long-context coding, perception, and agent planning","name":"MiniMax-M3"},"Qwen/Qwen3-235B-A22B":{"cost":{"input":0.2,"output":0.8},"description":"Large open Qwen MoE for multilingual reasoning, coding, and tool use","name":"Qwen3 235B-A22B"},"Qwen/Qwen3-235B-A22B-Instruct-2507":{"cost":{"input":0.855,"output":2.565},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3 235B-A22B Instruct 2507"},"Qwen/Qwen3-235B-A22B-Thinking-2507":{"cost":{"input":0.3,"output":3},"description":"Qwen reasoning model for deliberate problem solving, math, and coding","name":"Qwen3-235B-A22B-Thinking-2507"},"Qwen/Qwen3-32B":{"cost":{"input":0.29,"output":0.59},"description":"Dense open Qwen model for self-hosted chat, reasoning, and coding","name":"Qwen3 32B"},"Qwen/Qwen3-Coder-30B-A3B-Instruct":{"cost":{"input":0.07,"output":0.26},"description":"Smaller Qwen coder for efficient local agents and repo-level fixes","name":"Qwen3-Coder 30B-A3B Instruct"},"Qwen/Qwen3-Coder-480B-A35B-Instruct":{"cost":{"input":2,"output":2},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen3-Coder-480B-A35B-Instruct"},"Qwen/Qwen3-Coder-Next":{"cost":{"input":0.2,"output":1.5},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen3-Coder-Next"},"Qwen/Qwen3-Embedding-4B":{"cost":{"input":0.01,"output":0.0},"description":"Embedding model for semantic search, retrieval, clustering, and ranking pipelines","name":"Qwen 3 Embedding 4B"},"Qwen/Qwen3-Embedding-8B":{"cost":{"input":0.01,"output":0.0},"description":"Embedding model for semantic search, retrieval, clustering, and ranking pipelines","name":"Qwen 3 Embedding 8B"},"Qwen/Qwen3-Next-80B-A3B-Instruct":{"cost":{"input":0.25,"output":1.0},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3-Next-80B-A3B-Instruct"},"Qwen/Qwen3-Next-80B-A3B-Thinking":{"cost":{"input":0.3,"output":2.0},"description":"Qwen reasoning model for deliberate problem solving, math, and coding","name":"Qwen3-Next-80B-A3B-Thinking"},"Qwen/Qwen3.5-122B-A10B":{"cost":{"input":0.4,"output":3.2},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 122B-A10B"},"Qwen/Qwen3.5-27B":{"cost":{"input":0.3,"output":2.4},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 27B"},"Qwen/Qwen3.5-35B-A3B":{"cost":{"input":0.25,"output":2},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 35B-A3B"},"Qwen/Qwen3.5-397B-A17B":{"cost":{"input":0.6,"output":3.6},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5-397B-A17B"},"Qwen/Qwen3.5-9B":{"cost":{"input":0.17,"output":0.25},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3.5 9B"},"Qwen/Qwen3.6-27B":{"cost":{"input":0.47,"output":3.19},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.6 27B"},"Qwen/Qwen3.6-35B-A3B":{"cost":{"input":0.15,"output":0.95},"description":"Open multimodal Qwen MoE for local agents that need vision, audio, and code","name":"Qwen3.6 35B-A3B"},"XiaomiMiMo/MiMo-V2-Flash":{"cost":{"input":0.1,"output":0.3},"description":"MiMo flash model for fast multimodal assistance and agent workflows","name":"MiMo-V2-Flash"},"XiaomiMiMo/MiMo-V2.5":{"cost":{"input":0.4,"output":2},"description":"MiMo model for long-context reasoning, perception, and agentic tasks","name":"MiMo-V2.5"},"XiaomiMiMo/MiMo-V2.5-Pro":{"cost":{"input":1,"output":3},"description":"Stronger MiMo Pro tier for multimodal reasoning and coding-agent execution","name":"MiMo-V2.5-Pro"},"deepseek-ai/DeepSeek-R1":{"cost":{"input":0.7,"output":2.5},"description":"Classic open reasoning model for transparent math, coding, and deliberate problem solving","name":"DeepSeek-R1"},"deepseek-ai/DeepSeek-R1-0528":{"cost":{"input":3,"output":5},"description":"DeepSeek reasoning model for multi-step analysis, math, coding, and tools","name":"DeepSeek-R1-0528"},"deepseek-ai/DeepSeek-V3":{"cost":{"input":0.4,"output":1.3},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek-V3"},"deepseek-ai/DeepSeek-V3.1":{"cost":{"input":0.27,"output":1},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek-V3.1"},"deepseek-ai/DeepSeek-V3.2":{"cost":{"input":0.28,"output":0.4},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek-V3.2"},"deepseek-ai/DeepSeek-V4-Flash":{"cost":{"input":0.14,"output":0.28},"description":"Fast DeepSeek V4 lane for economical reasoning, coding, and long-context work","name":"DeepSeek V4 Flash"},"deepseek-ai/DeepSeek-V4-Flash-0731":{"cost":{"input":0.14,"output":0.28},"description":"Official DeepSeek V4 Flash release with enhanced agentic capabilities and integrated DSpark speculative decoding","name":"DeepSeek V4 Flash 0731"},"deepseek-ai/DeepSeek-V4-Pro":{"cost":{"cache_read":0.003625,"input":0.435,"output":0.87},"description":"Open MoE flagship with million-token context for coding and long agent runs","name":"DeepSeek V4 Pro"},"google/gemma-4-26B-A4B-it":{"cost":{"input":0.13,"output":0.4},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 4 26B A4B IT"},"google/gemma-4-31B-it":{"cost":{"input":0.14,"output":0.4},"description":"Largest Gemma 4 instruction model for open, self-hosted chat and reasoning","name":"Gemma 4 31B IT"},"meta-llama/Llama-3.3-70B-Instruct":{"cost":{"input":0.59,"output":0.79},"description":"Popular open Llama workhorse for multilingual chat, coding, and self-hosting","name":"Llama-3.3-70B-Instruct"},"moonshotai/Kimi-K2-Instruct":{"cost":{"input":1,"output":3},"description":"Kimi model for long-context chat, coding, and agentic reasoning","name":"Kimi-K2-Instruct"},"moonshotai/Kimi-K2-Instruct-0905":{"cost":{"input":1,"output":3},"description":"Kimi model for long-context chat, coding, and agentic reasoning","name":"Kimi-K2-Instruct-0905"},"moonshotai/Kimi-K2-Thinking":{"cost":{"cache_read":0.15,"input":0.6,"output":2.5},"description":"Kimi reasoning model for long-horizon research, planning, and tool use","name":"Kimi-K2-Thinking"},"moonshotai/Kimi-K2.5":{"cost":{"cache_read":0.1,"input":0.6,"output":3.0},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi-K2.5"},"moonshotai/Kimi-K2.6":{"cost":{"cache_read":0.16,"input":0.95,"output":4.0},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi-K2.6"},"moonshotai/Kimi-K2.7-Code":{"cost":{"input":0.95,"output":4},"description":"Coding-focused Kimi model, stronger on long-horizon repo work with less overthinking","name":"Kimi K2.7 Code"},"moonshotai/Kimi-K3":{"cost":{"input":3,"output":15},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi K3"},"openai/gpt-oss-120b":{"cost":{"input":0.25,"output":0.69},"description":"Open GPT reasoning model for self-hosted agents and controllable deployments","name":"GPT OSS 120B"},"openai/gpt-oss-20b":{"cost":{"input":0.1,"output":0.5},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"GPT OSS 20B"},"stepfun-ai/Step-3.5-Flash":{"cost":{"input":0.1,"output":0.3},"description":"StepFun flash lane for quick multimodal reasoning and coding assistance","name":"Step 3.5 Flash"},"stepfun-ai/Step-3.7-Flash":{"cost":{"input":0.2,"output":1.15},"description":"Newer StepFun flash model for faster agents, coding, and multimodal prompts","name":"Step 3.7 Flash"},"tencent/Hy3":{"cost":{"input":0.14,"output":0.58},"description":"Tencent Hy reasoning model for coding, instruction following, and agent tasks","name":"Hy3"},"thinkingmachines/Inkling":{"cost":{"input":1,"output":4.05},"description":"Multimodal model for analyzing text, images, documents, and rich media","name":"Inkling"},"thinkingmachines/Inkling-Small":{"cost":{"input":0.5,"output":1.2},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Inkling Small"},"zai-org/GLM-4.5":{"cost":{"input":0.6,"output":2.2},"description":"Hybrid-reasoning GLM release that made the 4.5 line broadly useful","name":"GLM-4.5"},"zai-org/GLM-4.5-Air":{"cost":{"input":0.13,"output":0.85},"description":"Lighter GLM-4.5 variant for fast coding assistance and cheaper agents","name":"GLM-4.5-Air"},"zai-org/GLM-4.5V":{"cost":{"input":0.6,"output":1.8},"description":"GLM vision model for visual reasoning, documents, and multimodal agents","name":"GLM-4.5V"},"zai-org/GLM-4.6":{"cost":{"input":0.55,"output":2.2},"description":"Late GLM-4 workhorse for coding agents, reasoning, and structured tasks","name":"GLM-4.6"},"zai-org/GLM-4.7":{"cost":{"cache_read":0.11,"input":0.6,"output":2.2},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM-4.7"},"zai-org/GLM-4.7-Flash":{"cost":{"input":0,"output":0},"description":"Efficient GLM model for fast reasoning, coding, and agent workflows","name":"GLM-4.7-Flash"},"zai-org/GLM-5":{"cost":{"cache_read":0.2,"input":1.0,"output":3.2},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM-5"},"zai-org/GLM-5.1":{"cost":{"cache_read":0.2,"input":1.0,"output":3.2},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM-5.1"},"zai-org/GLM-5.2":{"cost":{"input":1.4,"output":4.4},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM-5.2"}}},"minimax":{"models":{"MiniMax-M2":{"cost":{"input":0.3,"output":1.2},"description":"Efficient open MiniMax model built for coding agents and tool-heavy workflows","name":"MiniMax-M2"},"MiniMax-M2.1":{"cost":{"cache_read":0.03,"cache_write":0.375,"input":0.3,"output":1.2},"description":"Earlier MiniMax agent model for practical coding and productivity tasks","name":"MiniMax-M2.1"},"MiniMax-M2.5":{"cost":{"cache_read":0.03,"cache_write":0.375,"input":0.3,"output":1.2},"description":"Prior MiniMax coding model for agent workflows, office edits, and automation","name":"MiniMax-M2.5"},"MiniMax-M2.5-highspeed":{"cost":{"cache_read":0.06,"cache_write":0.375,"input":0.6,"output":2.4},"description":"High-speed MiniMax model for low-latency coding and agent workflows","name":"MiniMax-M2.5-highspeed"},"MiniMax-M2.7":{"cost":{"cache_read":0.06,"cache_write":0.375,"input":0.3,"output":1.2},"description":"Open MiniMax flagship for coding agents, office automation, and complex environments","name":"MiniMax-M2.7"},"MiniMax-M2.7-highspeed":{"cost":{"cache_read":0.06,"cache_write":0.375,"input":0.6,"output":2.4},"description":"Low-latency M2.7 variant for interactive coding plans and agent loops","name":"MiniMax-M2.7-highspeed"},"MiniMax-M3":{"cost":{"cache_read":0.06,"input":0.3,"output":1.2,"tiers":[{"cache_read":0.12,"input":0.6,"output":2.4,"tier":{"size":512000}}]},"description":"MiniMax multimodal model for long-context coding, perception, and agent planning","name":"MiniMax-M3"}}},"minimax_cn":{"models":{"MiniMax-M2":{"cost":{"input":0.3,"output":1.2},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"MiniMax-M2"},"MiniMax-M2.1":{"cost":{"cache_read":0.03,"cache_write":0.375,"input":0.3,"output":1.2},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"MiniMax-M2.1"},"MiniMax-M2.5":{"cost":{"cache_read":0.03,"cache_write":0.375,"input":0.3,"output":1.2},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"MiniMax-M2.5"},"MiniMax-M2.5-highspeed":{"cost":{"cache_read":0.06,"cache_write":0.375,"input":0.6,"output":2.4},"description":"High-speed MiniMax model for low-latency coding and agent workflows","name":"MiniMax-M2.5-highspeed"},"MiniMax-M2.7":{"cost":{"cache_read":0.06,"cache_write":0.375,"input":0.3,"output":1.2},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"MiniMax-M2.7"},"MiniMax-M2.7-highspeed":{"cost":{"cache_read":0.06,"cache_write":0.375,"input":0.6,"output":2.4},"description":"High-speed MiniMax model for low-latency coding and agent workflows","name":"MiniMax-M2.7-highspeed"},"MiniMax-M3":{"cost":{"cache_read":0.06,"input":0.3,"output":1.2,"tiers":[{"cache_read":0.12,"input":0.6,"output":2.4,"tier":{"size":512000}}]},"description":"MiniMax multimodal coding model for long-context reasoning and agent tasks","name":"MiniMax-M3"}}},"minimax_global":{"models":{"MiniMax-M2":{"cost":{"input":0.3,"output":1.2},"description":"Efficient open MiniMax model built for coding agents and tool-heavy workflows","name":"MiniMax-M2"},"MiniMax-M2.1":{"cost":{"cache_read":0.03,"cache_write":0.375,"input":0.3,"output":1.2},"description":"Earlier MiniMax agent model for practical coding and productivity tasks","name":"MiniMax-M2.1"},"MiniMax-M2.5":{"cost":{"cache_read":0.03,"cache_write":0.375,"input":0.3,"output":1.2},"description":"Prior MiniMax coding model for agent workflows, office edits, and automation","name":"MiniMax-M2.5"},"MiniMax-M2.5-highspeed":{"cost":{"cache_read":0.06,"cache_write":0.375,"input":0.6,"output":2.4},"description":"High-speed MiniMax model for low-latency coding and agent workflows","name":"MiniMax-M2.5-highspeed"},"MiniMax-M2.7":{"cost":{"cache_read":0.06,"cache_write":0.375,"input":0.3,"output":1.2},"description":"Open MiniMax flagship for coding agents, office automation, and complex environments","name":"MiniMax-M2.7"},"MiniMax-M2.7-highspeed":{"cost":{"cache_read":0.06,"cache_write":0.375,"input":0.6,"output":2.4},"description":"Low-latency M2.7 variant for interactive coding plans and agent loops","name":"MiniMax-M2.7-highspeed"},"MiniMax-M3":{"cost":{"cache_read":0.06,"input":0.3,"output":1.2,"tiers":[{"cache_read":0.12,"input":0.6,"output":2.4,"tier":{"size":512000}}]},"description":"MiniMax multimodal model for long-context coding, perception, and agent planning","name":"MiniMax-M3"}}},"mistral":{"models":{"codestral-latest":{"cost":{"input":0.3,"output":0.9},"description":"Mistral code model for completions, refactors, and developer IDE workflows","name":"Codestral (latest)"},"devstral-2512":{"cost":{"input":0.4,"output":2.0},"description":"Mistral's coding-agent model for repository work, terminal tasks, and software fixes","name":"Devstral 2"},"devstral-latest":{"cost":{"input":0.4,"output":2.0},"description":"Legacy model retained for compatibility with older integrations","name":"Devstral 2"},"devstral-medium-2507":{"cost":{"input":0.4,"output":2},"description":"Legacy model retained for compatibility with older integrations","name":"Devstral Medium"},"devstral-medium-latest":{"cost":{"input":0.4,"output":2},"description":"Legacy model retained for compatibility with older integrations","name":"Devstral 2 (latest)"},"devstral-small-2505":{"cost":{"input":0.1,"output":0.3},"description":"Legacy model retained for compatibility with older integrations","name":"Devstral Small 2505"},"devstral-small-2507":{"cost":{"input":0.1,"output":0.3},"description":"Legacy model retained for compatibility with older integrations","name":"Devstral Small"},"labs-devstral-small-2512":{"cost":{"input":0.0,"output":0.0},"description":"Legacy model retained for compatibility with older integrations","name":"Devstral Small 2"},"magistral-medium-latest":{"cost":{"input":2.0,"output":5.0},"description":"Mistral reasoning model for transparent analysis, math, and complex decisions","name":"Magistral Medium (latest)"},"magistral-small":{"cost":{"input":0.5,"output":1.5},"description":"Mistral reasoning model for transparent analysis, math, and complex decisions","name":"Magistral Small"},"ministral-3b-latest":{"cost":{"input":0.04,"output":0.04},"description":"Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads","name":"Ministral 3B (latest)"},"ministral-8b-latest":{"cost":{"input":0.1,"output":0.1},"description":"Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads","name":"Ministral 8B (latest)"},"mistral-embed":{"cost":{"input":0.1,"output":0.0},"description":"Embedding model for semantic search, retrieval, clustering, and ranking pipelines","name":"Mistral Embed"},"mistral-large-2411":{"cost":{"input":2.0,"output":6.0},"description":"Flagship Mistral model for advanced reasoning, coding, and multilingual work","name":"Mistral Large 2.1"},"mistral-large-2512":{"cost":{"input":0.5,"output":1.5},"description":"Mistral's largest general model for enterprise agents, coding, and multilingual reasoning","name":"Mistral Large 3"},"mistral-large-latest":{"cost":{"input":0.5,"output":1.5},"description":"Flagship Mistral model for advanced reasoning, coding, and multilingual work","name":"Mistral Large (latest)"},"mistral-medium-2505":{"cost":{"input":0.4,"output":2.0},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral Medium 3"},"mistral-medium-2508":{"cost":{"input":0.4,"output":2.0},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral Medium 3.1"},"mistral-medium-2604":{"cost":{"input":1.5,"output":7.5},"description":"Balanced Mistral model for enterprise assistants, multilingual work, and tools","name":"Mistral Medium 3.5"},"mistral-medium-latest":{"cost":{"input":1.5,"output":7.5},"description":"Balanced Mistral model for enterprise assistants, multilingual work, and tools","name":"Mistral Medium (latest)"},"mistral-nemo":{"cost":{"input":0.15,"output":0.15},"description":"Efficient Mistral-NVIDIA open model for multilingual chat and local deployment","name":"Mistral Nemo"},"mistral-small-2506":{"cost":{"input":0.1,"output":0.3},"description":"Efficient Mistral model for fast chat, extraction, and production assistants","name":"Mistral Small 3.2"},"mistral-small-2603":{"cost":{"input":0.15,"output":0.6},"description":"Fast Mistral production model for chat, extraction, and cost-sensitive agents","name":"Mistral Small 4"},"mistral-small-latest":{"cost":{"input":0.15,"output":0.6},"description":"Efficient Mistral model for fast chat, extraction, and production assistants","name":"Mistral Small (latest)"},"open-mistral-7b":{"cost":{"input":0.25,"output":0.25},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral 7B"},"open-mistral-nemo":{"cost":{"input":0.15,"output":0.15},"description":"Legacy model retained for compatibility with older integrations","name":"Open Mistral Nemo"},"open-mixtral-8x22b":{"cost":{"input":2.0,"output":6.0},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mixtral 8x22B"},"open-mixtral-8x7b":{"cost":{"input":0.7,"output":0.7},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mixtral 8x7B"},"pixtral-12b":{"cost":{"input":0.15,"output":0.15},"description":"Mistral vision-language model for image understanding and multimodal chat","name":"Pixtral 12B"},"pixtral-large-latest":{"cost":{"input":2.0,"output":6.0},"description":"Mistral's larger vision model for document-heavy image understanding and chat","name":"Pixtral Large (latest)"},"voxtral-mini-latest":{"description":"Speech transcription model for accurate audio-to-text and captioning workflows","name":"Voxtral Mini (latest)"},"voxtral-mini-tts-latest":{"description":"Multilingual text-to-speech model with zero-shot voice cloning","name":"Voxtral Mini TTS (latest)"},"voxtral-small-latest":{"cost":{"input":0.1,"output":0.3},"description":"Instruct model with native audio input for speech understanding and tool use","name":"Voxtral Small (latest)"}}},"moonshot":{"models":{"kimi-k2-0711-preview":{"cost":{"cache_read":0.15,"input":0.6,"output":2.5},"description":"Kimi model for long-context chat, coding, and agentic reasoning","name":"Kimi K2 0711"},"kimi-k2-0905-preview":{"cost":{"cache_read":0.15,"input":0.6,"output":2.5},"description":"Kimi model for long-context chat, coding, and agentic reasoning","name":"Kimi K2 0905"},"kimi-k2-thinking":{"cost":{"cache_read":0.15,"input":0.6,"output":2.5},"description":"Thinking Kimi model for slower research passes, planning, and hard technical questions","name":"Kimi K2 Thinking"},"kimi-k2-thinking-turbo":{"cost":{"cache_read":0.15,"input":1.15,"output":8},"description":"Kimi reasoning model for long-horizon research, planning, and tool use","name":"Kimi K2 Thinking Turbo"},"kimi-k2-turbo-preview":{"cost":{"cache_read":0.6,"input":2.4,"output":10},"description":"Fast Kimi model for responsive chat, coding help, and agent loops","name":"Kimi K2 Turbo"},"kimi-k2.5":{"cost":{"cache_read":0.1,"input":0.6,"output":3.0},"description":"Earlier Kimi frontier model for long-context agents, coding, and multimodal work","name":"Kimi K2.5"},"kimi-k2.6":{"cost":{"cache_read":0.16,"input":0.95,"output":4.0},"description":"Multimodal Kimi workhorse for agent loops, coding tasks, and visual context","name":"Kimi K2.6"},"kimi-k2.7-code":{"cost":{"cache_read":0.19,"input":0.95,"output":4.0},"description":"Coding-focused Kimi model, stronger on long-horizon repo work with less overthinking","name":"Kimi K2.7 Code"},"kimi-k2.7-code-highspeed":{"cost":{"cache_read":0.38,"input":1.9,"output":8.0},"description":"Coding-focused Kimi model, stronger on long-horizon repo work with less overthinking","name":"Kimi K2.7 Code HighSpeed"},"kimi-k3":{"cost":{"cache_read":0.3,"input":3.0,"output":15.0},"description":"Multimodal Kimi model with 1M context and toggleable max-effort thinking for long-horizon agent work","name":"Kimi K3"}}},"morph":{"models":{"auto":{"cost":{"input":0.85,"output":1.55},"description":"Automatic model router for matching prompts to suitable backends and budgets","name":"Auto"},"morph-v3-fast":{"cost":{"input":0.8,"output":1.2},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Morph v3 Fast"},"morph-v3-large":{"cost":{"input":0.9,"output":1.9},"description":"Flagship model for demanding analysis, coding, and production agent workflows","name":"Morph v3 Large"}}},"nano-gpt":{"models":{"Baichuan-M2":{"cost":{"cache_read":7.865,"input":15.73,"output":15.73},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Baichuan M2 32B Medical"},"Baichuan4-Air":{"cost":{"cache_read":0.0785,"input":0.157,"output":0.157},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Baichuan 4 Air"},"Baichuan4-Turbo":{"cost":{"cache_read":1.21,"input":2.42,"output":2.42},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Baichuan 4 Turbo"},"Doctor-Shotgun/MS3.2-24B-Magnum-Diamond":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"MS3.2 24B Magnum Diamond"},"EVA-UNIT-01/EVA-LLaMA-3.33-70B-v0.0":{"cost":{"cache_read":1.003,"input":2.006,"output":2.006},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"EVA Llama 3.33 70B"},"EVA-UNIT-01/EVA-LLaMA-3.33-70B-v0.1":{"cost":{"cache_read":1.003,"input":2.006,"output":2.006},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"EVA-LLaMA-3.33-70B-v0.1"},"EVA-UNIT-01/EVA-Qwen2.5-32B-v0.2":{"cost":{"cache_read":0.3995,"input":0.799,"output":0.799},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"EVA-Qwen2.5-32B-v0.2"},"EVA-UNIT-01/EVA-Qwen2.5-72B-v0.2":{"cost":{"cache_read":0.3995,"input":0.799,"output":0.799},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"EVA-Qwen2.5-72B-v0.2"},"Envoid/Llama-3.05-NT-Storybreaker-Ministral-70B":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads","name":"Llama 3.05 Storybreaker Ministral 70b"},"Envoid/Llama-3.05-Nemotron-Tenyxchat-Storybreaker-70B":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"Nemotron model for efficient reasoning, coding, and specialized AI agents","name":"Nemotron Tenyxchat Storybreaker 70b"},"GLM-4.6-Derestricted-v5":{"cost":{"cache_read":0.2,"input":0.4,"output":1.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GLM 4.6 Derestricted v5"},"GalrionSoftworks/MN-LooseCannon-12B-v1":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"MN-LooseCannon-12B-v1"},"Gemma-4-31B-Claude-4.6-Opus-Reasoning-Distilled":{"cost":{"cache_read":0.0306,"input":0.306,"output":0.306},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"Gemma 4 31B Claude 4.6 Opus Reasoning Distilled"},"Gemma-4-31B-Cognitive-Unshackled":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemma 4 31B Cognitive Unshackled"},"Gemma-4-31B-DarkIdol":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemma 4 31B DarkIdol"},"Gemma-4-31B-GarnetV2":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemma 4 31B Garnet V2"},"Gemma-4-31B-Gemopus":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemma 4 31B Gemopus"},"Gemma-4-31B-Musica-v1":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemma 4 31B Musica v1"},"Gemma-4-31B-Queen":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemma 4 31B Queen"},"Gryphe/MythoMax-L2-13b":{"cost":{"cache_read":0.05015,"input":0.1003,"output":0.1003},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"MythoMax 13B"},"LLM360/K2-Think":{"cost":{"cache_read":0.085,"input":0.17,"output":0.68},"description":"Kimi reasoning model for long-horizon research, planning, and tool use","name":"K2-Think"},"LatitudeGames/Wayfarer-Large-70B-Llama-3.3":{"cost":{"cache_read":0.35,"input":0.7,"output":0.7},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.3 70B Wayfarer"},"MarinaraSpaghetti/NemoMix-Unleashed-12B":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"NemoMix 12B Unleashed"},"Meta-Llama-3-1-8B-Instruct-FP8":{"cost":{"cache_read":0.01,"input":0.02,"output":0.03},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Llama 3.1 8B (decentralized)"},"MiniMax-M1":{"cost":{"cache_read":0.0697,"input":0.1394,"output":1.3328},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"MiniMax M1"},"MiniMax-M2":{"cost":{"cache_read":0.085,"input":0.17,"output":1.53},"description":"Efficient open MiniMax model built for coding agents and tool-heavy workflows","name":"MiniMax M2"},"MiniMaxAI/MiniMax-M1-80k":{"cost":{"cache_read":0.3026,"input":0.6052,"output":2.4225},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"MiniMax M1 80K"},"NeverSleep/Lumimaid-v0.2-70B":{"cost":{"cache_read":0.5,"input":1,"output":1.5},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Lumimaid v0.2"},"NousResearch/Hermes-4-70B:thinking":{"cost":{"cache_read":0.1003,"input":0.2006,"output":0.3995},"description":"General-purpose chat model for instruction following, writing, and analysis","name":"Hermes 4 (Thinking)"},"NousResearch/hermes-3-llama-3.1-70b":{"cost":{"cache_read":0.204,"input":0.408,"output":0.408},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Hermes 3 70B"},"NousResearch/hermes-4-405b":{"cost":{"cache_read":0.15,"input":0.3,"output":1.2},"description":"Flagship model for demanding analysis, coding, and production agent workflows","name":"Hermes 4 Large"},"NousResearch/hermes-4-405b:thinking":{"cost":{"cache_read":0.15,"input":0.3,"output":1.2},"description":"Flagship model for demanding analysis, coding, and production agent workflows","name":"Hermes 4 Large (Thinking)"},"NousResearch/hermes-4-70b":{"cost":{"cache_read":0.1003,"input":0.2006,"output":0.3995},"description":"General-purpose chat model for instruction following, writing, and analysis","name":"Hermes 4 Medium"},"Qwen3.5-27B-Anko":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Anko"},"Qwen3.5-27B-BlueStar-Derestricted":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B BlueStar Derestricted"},"Qwen3.5-27B-BlueStar-Derestricted-Lite":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B BlueStar Derestricted Lite"},"Qwen3.5-27B-BlueStar-v2-Derestricted":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B BlueStar v2 Derestricted"},"Qwen3.5-27B-BlueStar-v2-Derestricted-Lite":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B BlueStar v2 Derestricted Lite"},"Qwen3.5-27B-BlueStar-v3-Derestricted":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B BlueStar v3 Derestricted"},"Qwen3.5-27B-BlueStar-v3-Derestricted-Lite":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B BlueStar v3 Derestricted Lite"},"Qwen3.5-27B-Derestricted":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Derestricted"},"Qwen3.5-27B-Infracelestial":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Infracelestial"},"Qwen3.5-27B-Marvin-DPO-V2-Derestricted":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Marvin DPO V2 Derestricted"},"Qwen3.5-27B-Marvin-DPO-V2-Derestricted-Lite":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Marvin DPO V2 Derestricted Lite"},"Qwen3.5-27B-Marvin-V2-Derestricted":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Marvin V2 Derestricted"},"Qwen3.5-27B-Marvin-V2-Derestricted-Lite":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Marvin V2 Derestricted Lite"},"Qwen3.5-27B-Musica-v1":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Musica v1"},"Qwen3.5-27B-NaNovel-Derestricted":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B NaNovel Derestricted"},"Qwen3.5-27B-NaNovel-Derestricted-Lite":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B NaNovel Derestricted Lite"},"Qwen3.5-27B-Omega-Evolution-v2.0-Derestricted":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Omega Evolution v2.0 Derestricted"},"Qwen3.5-27B-Omega-Evolution-v2.0-Derestricted-Lite":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Omega Evolution v2.0 Derestricted Lite"},"Qwen3.5-27B-Omega-Evolution-v2.2-Derestricted":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Omega Evolution v2.2 Derestricted"},"Qwen3.5-27B-Omega-Evolution-v2.2-Derestricted-Lite":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Omega Evolution v2.2 Derestricted Lite"},"Qwen3.5-27B-Queen-Derestricted":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Queen Derestricted"},"Qwen3.5-27B-Queen-Derestricted-Lite":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Queen Derestricted Lite"},"Qwen3.5-27B-RpRMax-v1":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B RpRMax v1"},"Qwen3.5-27B-Vivid-Durian":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Vivid Durian"},"Qwen3.5-27B-Writer-Derestricted":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Writer Derestricted"},"Qwen3.5-27B-Writer-Derestricted-Lite":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Writer Derestricted Lite"},"Qwen3.5-27B-Writer-V2-Derestricted":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Writer V2 Derestricted"},"Qwen3.5-27B-Writer-V2-Derestricted-Lite":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B Writer V2 Derestricted Lite"},"Qwen3.5-27B-earica-Derestricted":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B earica Derestricted"},"Qwen3.5-27B-earica-Derestricted-Lite":{"cost":{"cache_read":0.153,"input":0.306,"output":0.306},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3.5 27B earica Derestricted Lite"},"ReadyArt/MS3.2-The-Omega-Directive-24B-Unslop-v2.0":{"cost":{"cache_read":0.25,"input":0.5,"output":0.5},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Omega Directive 24B Unslop v2.0"},"Salesforce/Llama-xLAM-2-70b-fc-r":{"cost":{"cache_read":1.25,"input":2.5,"output":2.5},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama-xLAM-2 70B fc-r"},"Sao10K/L3-8B-Stheno-v3.2":{"cost":{"cache_read":0.1003,"input":0.2006,"output":0.2006},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Sao10K Stheno 8b"},"Sao10K/L3.1-70B-Euryale-v2.2":{"cost":{"cache_read":0.153,"input":0.306,"output":0.357},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.1 70B Euryale"},"Sao10K/L3.1-70B-Hanami-x1":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.1 70B Hanami"},"Sao10K/L3.3-70B-Euryale-v2.3":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.3 70B Euryale"},"Steelskull/L3.3-Cu-Mai-R1-70b":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.3 70B Cu Mai"},"Steelskull/L3.3-Electra-R1-70b":{"cost":{"cache_read":0.349945,"input":0.69989,"output":0.69989},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Steelskull Electra R1 70b"},"Steelskull/L3.3-MS-Evayale-70B":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Evayale 70b "},"Steelskull/L3.3-MS-Nevoria-70b":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Steelskull Nevoria 70b"},"Steelskull/L3.3-Nevoria-R1-70b":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Steelskull Nevoria R1 70b"},"TEE/deepseek-v3.1":{"cost":{"cache_read":0.5,"input":1,"output":2.5},"description":"Hybrid-reasoning DeepSeek model with thinking and non-thinking modes","name":"DeepSeek V3.1 TEE"},"TEE/deepseek-v3.2":{"cost":{"cache_read":0.25,"input":0.5,"output":1},"description":"Hybrid-reasoning DeepSeek model with thinking and non-thinking modes, sparse attention, and tool-use","name":"DeepSeek V3.2 TEE"},"TEE/deepseek-v4-flash":{"cost":{"cache_read":0.04,"input":0.2,"output":0.4},"description":"Fast DeepSeek V4 lane for economical reasoning, coding, and long-context work","name":"DeepSeek V4 Flash TEE"},"TEE/gemma-3-27b-it":{"cost":{"cache_read":0.1,"input":0.2,"output":0.8},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 3 27B TEE"},"TEE/gemma-4-26b-a4b-uncensored":{"cost":{"cache_read":0.075,"input":0.15,"output":0.7},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 4 26B A4B Uncensored TEE"},"TEE/gemma-4-31b-it":{"cost":{"cache_read":0.075,"input":0.15,"output":0.46},"description":"Largest Gemma 4 instruction model for open, self-hosted chat and reasoning","name":"Gemma 4 31B IT TEE"},"TEE/gemma4-31b":{"cost":{"cache_read":0.45,"input":0.45,"output":1},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 4 31B"},"TEE/gemma4-31b:thinking":{"cost":{"cache_read":0.45,"input":0.45,"output":1},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 4 31B Thinking TEE"},"TEE/glm-4.7":{"cost":{"cache_read":0.425,"input":0.85,"output":3.3},"description":"Mature GLM model for dependable coding, reasoning, and structured agent tasks","name":"GLM 4.7 TEE"},"TEE/glm-5.1":{"cost":{"cache_read":0.3,"input":1.5,"output":5.25},"description":"Strong GLM coding model for agentic engineering, terminals, and repository generation","name":"GLM 5.1 TEE"},"TEE/glm-5.1-thinking":{"cost":{"cache_read":0.3,"input":1.5,"output":5.25},"description":"Strong GLM coding model for agentic engineering, terminals, and repository generation","name":"GLM 5.1 Thinking TEE"},"TEE/glm-5.2":{"cost":{"cache_read":0.5,"input":1.4,"output":4.6},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM 5.2 TEE"},"TEE/glm-5.2:thinking":{"cost":{"cache_read":0.5,"input":1.4,"output":4.6},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM 5.2 Thinking TEE"},"TEE/gpt-oss-120b":{"cost":{"cache_read":2,"input":2,"output":2},"description":"Open GPT reasoning model for self-hosted agents and controllable deployments","name":"GPT-OSS 120B TEE"},"TEE/gpt-oss-20b":{"cost":{"cache_read":0.1,"input":0.2,"output":0.8},"description":"Open GPT reasoning model for self-hosted agents and controllable deployments","name":"GPT-OSS 20B TEE"},"TEE/kimi-k2.5":{"cost":{"cache_read":0.3,"input":0.6,"output":3},"description":"Earlier Kimi frontier model for long-context agents, coding, and multimodal work","name":"Kimi K2.5 TEE"},"TEE/kimi-k2.5-thinking":{"cost":{"cache_read":0.3,"input":0.6,"output":3},"description":"Earlier Kimi frontier model for long-context agents, coding, and multimodal work","name":"Kimi K2.5 Thinking TEE"},"TEE/kimi-k2.6":{"cost":{"cache_read":0.375,"input":1.5,"output":5.25},"description":"Multimodal Kimi workhorse for agent loops, coding tasks, and visual context","name":"Kimi K2.6 TEE"},"TEE/kimi-k3":{"cost":{"cache_read":1.5,"input":3,"output":15},"description":"Multimodal Kimi model with 1M context and toggleable max-effort thinking for long-horizon agent work","name":"Kimi K3 TEE"},"TEE/llama3-3-70b":{"cost":{"cache_read":2,"input":2,"output":2},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.3 70B"},"TEE/minimax-m2.5":{"cost":{"cache_read":0.1,"input":0.2,"output":1.38},"description":"Prior MiniMax coding model for agent workflows, office edits, and automation","name":"MiniMax M2.5 TEE"},"TEE/qwen2.5-vl-72b-instruct":{"cost":{"cache_read":0.35,"input":0.7,"output":0.7},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen2.5 VL 72B TEE"},"TEE/qwen3.5-122b-a10b":{"cost":{"cache_read":0.23,"input":0.46,"output":3.68},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 122B A10B TEE"},"TEE/qwen3.5-27b":{"cost":{"cache_read":0.15,"input":0.3,"output":2.4},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 27B TEE"},"TEE/qwen3.5-397b-a17b":{"cost":{"cache_read":0.275,"input":0.55,"output":3.5},"description":"Large open Qwen multimodal MoE for visual agents and long technical tasks","name":"Qwen3.5 397B A17B TEE"},"TEE/qwen3.6-27b":{"cost":{"cache_read":0.16,"input":0.32,"output":2.7},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.6 27B TEE"},"TEE/qwen3.6-35b-a3b":{"cost":{"cache_read":0.1,"input":0.2,"output":1.27},"description":"Open multimodal Qwen MoE for local agents that need vision, audio, and code","name":"Qwen3.6 35B A3B TEE"},"TEE/qwen3.6-35b-a3b-uncensored":{"cost":{"cache_read":0.15,"input":0.3,"output":1.5},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Qwen3.6 35B A3B Uncensored TEE"},"THUDM/GLM-4-32B-0414":{"cost":{"cache_read":0.1,"input":0.2,"output":0.2},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM 4 32B 0414"},"THUDM/GLM-4-9B-0414":{"cost":{"cache_read":0.1,"input":0.2,"output":0.2},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM 4 9B 0414"},"THUDM/GLM-Z1-9B-0414":{"cost":{"cache_read":0.1,"input":0.2,"output":0.2},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM Z1 9B 0414"},"TheDrummer/Anubis-70B-v1":{"cost":{"cache_read":0.155,"input":0.31,"output":0.31},"description":"General-purpose chat model for instruction following, writing, and analysis","name":"Anubis 70B v1"},"TheDrummer/Anubis-70B-v1.1":{"cost":{"cache_read":0.155,"input":0.31,"output":0.31},"description":"General-purpose chat model for instruction following, writing, and analysis","name":"Anubis 70B v1.1"},"TheDrummer/Cydonia-24B-v2":{"cost":{"cache_read":0.05015,"input":0.1003,"output":0.1207},"description":"General-purpose chat model for instruction following, writing, and analysis","name":"The Drummer Cydonia 24B v2"},"TheDrummer/Cydonia-24B-v4":{"cost":{"cache_read":0.1003,"input":0.2006,"output":0.2414},"description":"General-purpose chat model for instruction following, writing, and analysis","name":"The Drummer Cydonia 24B v4"},"TheDrummer/Cydonia-24B-v4.1":{"cost":{"cache_read":0.16,"input":0.35,"output":0.55},"description":"General-purpose chat model for instruction following, writing, and analysis","name":"The Drummer Cydonia 24B v4.1"},"TheDrummer/Cydonia-24B-v4.3":{"cost":{"cache_read":0.06,"input":0.12,"output":0.15},"description":"General-purpose chat model for instruction following, writing, and analysis","name":"The Drummer Cydonia 24B v4.3"},"TheDrummer/Magidonia-24B-v4.3":{"cost":{"cache_read":0.05015,"input":0.1003,"output":0.1207},"description":"General-purpose chat model for instruction following, writing, and analysis","name":"The Drummer Magidonia 24B v4.3"},"TheDrummer/Rocinante-12B-v1.1":{"cost":{"cache_read":0.204,"input":0.408,"output":0.595},"description":"General-purpose chat model for instruction following, writing, and analysis","name":"Rocinante 12b"},"TheDrummer/UnslopNemo-12B-v4.1":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"Multimodal model for analyzing text, images, documents, and rich media","name":"UnslopNemo 12b v4"},"TheDrummer/skyfall-36b-v2":{"cost":{"cache_read":0.25,"input":0.55,"output":0.8},"description":"Multimodal model for analyzing text, images, documents, and rich media","name":"TheDrummer Skyfall 36B V2"},"Tongyi-Zhiwen/QwenLong-L1-32B":{"cost":{"cache_read":0.07,"input":0.14,"output":0.6},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"QwenLong L1 32B"},"VongolaChouko/Starcannon-Unleashed-12B-v1.0":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral Nemo Starcannon 12b v1"},"abacusai/Dracarys-72B-Instruct":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.1 70B Dracarys 2"},"aion-labs/aion-2.0":{"cost":{"cache_read":0.2,"input":0.8,"output":1.6},"description":"General-purpose chat model for instruction following, writing, and analysis","name":"AionLabs: Aion-2.0"},"aion-labs/aion-2.5":{"cost":{"cache_read":0.35,"input":1,"output":3},"description":"General-purpose chat model for instruction following, writing, and analysis","name":"AionLabs: Aion-2.5"},"aion-labs/aion-3.0":{"cost":{"cache_read":0.75,"input":3,"output":6},"description":"Aion 3.0 is a GLM-family collaborative generation model tuned for immersive roleplay and storytelling, with stronger narrative structure, tension, conflict, and nuanced mature themes.","name":"AionLabs: Aion 3.0"},"aion-labs/aion-3.0-mini":{"cost":{"cache_read":0.18,"input":0.7,"output":1.4},"description":"Aion 3.0 Mini is a DeepSeek-family collaborative generation model tuned for immersive roleplay and storytelling, with stronger narrative structure, tension, conflict, and nuanced mature themes.","name":"AionLabs: Aion 3.0 Mini"},"aion-labs/aion-rp-llama-3.1-8b":{"cost":{"cache_read":0.4,"input":0.8,"output":1.6},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.1 8b (uncensored)"},"alibaba/qwen3.6-27b":{"cost":{"cache_read":0.1015,"input":0.203,"output":2.24},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.6 27B"},"alibaba/qwen3.6-27b:thinking":{"cost":{"cache_read":0.1015,"input":0.203,"output":2.24},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.6 27B Thinking"},"alibaba/qwen3.6-flash":{"cost":{"cache_read":0.02,"cache_write":0.24,"input":0.19,"output":1.16},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.6 Flash"},"amazon/nova-2-lite-v1":{"cost":{"cache_read":0.255,"input":0.51,"output":4.25},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Amazon Nova 2 Lite"},"amazon/nova-lite-v1":{"cost":{"cache_read":0.02975,"input":0.0595,"output":0.238},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Amazon Nova Lite 1.0"},"amazon/nova-micro-v1":{"cost":{"cache_read":0.01785,"input":0.0357,"output":0.1394},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Amazon Nova Micro 1.0"},"amazon/nova-pro-v1":{"cost":{"cache_read":0.3995,"input":0.799,"output":3.196},"description":"Flagship model for demanding analysis, coding, and production agent workflows","name":"Amazon Nova Pro 1.0"},"anthracite-org/magnum-v2-72b":{"cost":{"cache_read":1.003,"input":2.006,"output":2.992},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Magnum V2 72B"},"anthracite-org/magnum-v4-72b":{"cost":{"cache_read":1.003,"input":2.006,"output":2.992},"description":"Open Llama multimodal model for image understanding and text reasoning","name":"Magnum v4 72B"},"anthropic/claude-fable-5":{"cost":{"cache_read":1,"cache_write":12.5,"input":10,"output":50},"description":"Claude model for creative writing, analysis, and controlled agent workflows","name":"Claude Fable 5"},"anthropic/claude-fable-latest":{"cost":{"cache_read":1,"cache_write":12.5,"input":10,"output":50},"description":"Compatibility alias for Claude Fable.","name":"Claude Fable Latest"},"anthropic/claude-haiku-latest":{"cost":{"cache_read":0.1,"input":1,"output":5},"description":"Fast Claude model for responsive assistance, classification, and lightweight agents","name":"Claude Haiku Latest"},"anthropic/claude-opus-4.6":{"cost":{"cache_read":0.5,"input":5,"output":25},"description":"High-end Claude for difficult coding, planning, and slower expert reasoning","name":"Claude 4.6 Opus"},"anthropic/claude-opus-4.6:thinking":{"cost":{"cache_read":0.5,"input":5,"output":25},"description":"High-end Claude for difficult coding, planning, and slower expert reasoning","name":"Claude 4.6 Opus Thinking"},"anthropic/claude-opus-4.6:thinking:low":{"cost":{"cache_read":0.5,"input":5,"output":25},"description":"High-end Claude for difficult coding, planning, and slower expert reasoning","name":"Claude 4.6 Opus Thinking Low"},"anthropic/claude-opus-4.6:thinking:max":{"cost":{"cache_read":0.5,"input":5,"output":25},"description":"High-end Claude for difficult coding, planning, and slower expert reasoning","name":"Claude 4.6 Opus Thinking Max"},"anthropic/claude-opus-4.6:thinking:medium":{"cost":{"cache_read":0.5,"input":5,"output":25},"description":"High-end Claude for difficult coding, planning, and slower expert reasoning","name":"Claude 4.6 Opus Thinking Medium"},"anthropic/claude-opus-4.7":{"cost":{"cache_read":0.5,"input":5,"output":25},"description":"Stronger Opus tier for advanced software work and high-stakes reasoning","name":"Claude 4.7 Opus"},"anthropic/claude-opus-4.7:thinking":{"cost":{"cache_read":0.5,"input":5,"output":25},"description":"Stronger Opus tier for advanced software work and high-stakes reasoning","name":"Claude 4.7 Opus Thinking"},"anthropic/claude-opus-4.8":{"cost":{"cache_read":0.5,"input":5,"output":25},"description":"Top Claude Opus tier for the hardest reasoning, coding, and long-horizon agents","name":"Claude Opus 4.8"},"anthropic/claude-opus-4.8:thinking":{"cost":{"cache_read":0.5,"input":5,"output":25},"description":"Top Claude Opus tier for the hardest reasoning, coding, and long-horizon agents","name":"Claude Opus 4.8 Thinking"},"anthropic/claude-opus-5":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Strongest Claude Opus model for coding, agents, and professional work","name":"Claude Opus 5"},"anthropic/claude-opus-latest":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus Latest"},"anthropic/claude-sonnet-4.6":{"cost":{"cache_read":0.3,"input":3,"output":15},"description":"Claude workhorse for coding agents, careful analysis, and production cost control","name":"Claude Sonnet 4.6"},"anthropic/claude-sonnet-4.6:thinking":{"cost":{"cache_read":0.3,"input":3,"output":15},"description":"Claude workhorse for coding agents, careful analysis, and production cost control","name":"Claude Sonnet 4.6 Thinking"},"anthropic/claude-sonnet-5":{"cost":{"cache_read":0.2,"cache_write":2.5,"input":2,"output":10},"description":"Everyday Claude agent model for coding, planning, browsing, and general work","name":"Claude Sonnet 5"},"anthropic/claude-sonnet-5:thinking":{"cost":{"cache_read":0.2,"cache_write":2.5,"input":2,"output":10},"description":"Everyday Claude agent model for coding, planning, browsing, and general work","name":"Claude Sonnet 5 Thinking"},"anthropic/claude-sonnet-latest":{"cost":{"cache_read":0.2,"cache_write":2.5,"input":2,"output":10},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude Sonnet Latest"},"arcee-ai/trinity-large-thinking":{"cost":{"cache_read":0.125,"input":0.25,"output":0.9},"description":"Flagship model for demanding analysis, coding, and production agent workflows","name":"Trinity Large Thinking"},"asi1-mini":{"cost":{"cache_read":0.5,"input":1,"output":1},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"ASI1 Mini"},"auto-model":{"cost":{"input":0,"output":0},"description":"Automatic model router for matching prompts to suitable backends and budgets","name":"Auto model"},"auto-model-basic":{"cost":{"cache_read":4.998,"input":9.996,"output":19.992},"description":"Automatic model router for matching prompts to suitable backends and budgets","name":"Auto model (Basic)"},"auto-model-premium":{"cost":{"cache_read":4.998,"input":9.996,"output":19.992},"description":"Automatic model router for matching prompts to suitable backends and budgets","name":"Auto model (Premium)"},"auto-model-standard":{"cost":{"cache_read":4.998,"input":9.996,"output":19.992},"description":"Automatic model router for matching prompts to suitable backends and budgets","name":"Auto model (Standard)"},"azure-gpt-4-turbo":{"cost":{"input":10,"output":30},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Azure gpt-4-turbo"},"azure-gpt-4o":{"cost":{"cache_read":1.25,"input":2.5,"output":10},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Azure gpt-4o"},"azure-gpt-4o-mini":{"cost":{"cache_read":0.075,"input":0.15,"output":0.6},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Azure gpt-4o-mini"},"azure-o1":{"cost":{"cache_read":7.5,"input":15,"output":60},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"Azure o1"},"azure-o3-mini":{"cost":{"cache_read":0.55,"input":1.1,"output":4.4},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"Azure o3-mini"},"baseten/Kimi-K2-Instruct-FP4":{"cost":{"cache_read":0.2,"input":0.4,"output":1.8},"description":"Kimi model for long-context chat, coding, and agentic reasoning","name":"Kimi K2 0711 Instruct FP4"},"brave":{"cost":{"input":5,"output":5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Brave (Answers)"},"brave-pro":{"cost":{"input":5,"output":5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Brave (Pro)"},"brave-research":{"cost":{"input":5,"output":5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Brave (Research)"},"bytedance-seed/seed-2.0-lite":{"cost":{"cache_read":0.125,"input":0.25,"output":2},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"ByteDance Seed 2.0 Lite"},"bytedance/doubao-seed-2.1-pro":{"cost":{"cache_read":0.5,"input":1,"output":5},"description":"Higher-capability model in the Doubao Seed 2.1 family for agentic coding, long-context analysis, complex instruction following, and productivity workflows. Supports a 256k context window and up to 128k output tokens. Note: privacy and logging guarantees may be limited.","name":"Doubao Seed 2.1 Pro"},"bytedance/doubao-seed-2.1-turbo":{"cost":{"cache_read":0.25,"input":0.5,"output":2.5},"description":"Fast, lower-cost model in the Doubao Seed 2.1 family for everyday chat, coding assistance, document work, and high-throughput productivity tasks. Supports a 256k context window and up to 128k output tokens. Note: privacy and logging guarantees may be limited.","name":"Doubao Seed 2.1 Turbo"},"bytedance/doubao-seed-character":{"cost":{"cache_read":0.0236,"cache_write":0.0025,"input":0.1179,"output":0.2947},"description":"ByteDance's character-focused Doubao Seed model for roleplay, persona consistency, dialogue, and creative character interactions. It supports text and image input with a 128k context window. Requests route through ZenMux to ByteDance; ZenMux does not publish a model-API zero-retention or training guarantee, so avoid sensitive data.","name":"Doubao Seed Character"},"celeris-1":{"cost":{"cache_read":1,"input":2,"output":6},"description":"Celeris 1 is a diffusion language model built for ultra-low-latency classification, extraction, judging, query rewriting, and other short structured responses.","name":"Celeris 1"},"chutesai/Mistral-Small-3.2-24B-Instruct-2506":{"cost":{"cache_read":0.1,"input":0.2,"output":0.4},"description":"Efficient Mistral model for fast chat, extraction, and production assistants","name":"Mistral Small 3.2 24b Instruct"},"claude-haiku-4-5-20251001":{"cost":{"cache_read":0.1,"input":1,"output":5},"description":"Fast Claude model for responsive assistance, classification, and lightweight agents","name":"Claude Haiku 4.5"},"claude-haiku-4-5-20251001-thinking":{"cost":{"cache_read":0.1,"input":1,"output":5},"description":"Fast Claude model for responsive assistance, classification, and lightweight agents","name":"Claude Haiku 4.5 Thinking"},"claude-opus-4-1-20250805":{"cost":{"cache_read":1.5,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude 4.1 Opus"},"claude-opus-4-1-thinking":{"cost":{"cache_read":1.5,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude 4.1 Opus Thinking"},"claude-opus-4-1-thinking:1024":{"cost":{"cache_read":1.5,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude 4.1 Opus Thinking (1K)"},"claude-opus-4-1-thinking:32000":{"cost":{"cache_read":1.5,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude 4.1 Opus Thinking (32K)"},"claude-opus-4-1-thinking:32768":{"cost":{"cache_read":1.5,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude 4.1 Opus Thinking (32K)"},"claude-opus-4-1-thinking:8192":{"cost":{"cache_read":1.5,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude 4.1 Opus Thinking (8K)"},"claude-opus-4-20250514":{"cost":{"cache_read":1.5,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude 4 Opus"},"claude-opus-4-5-20251101":{"cost":{"cache_read":0.5,"input":5,"output":25},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude 4.5 Opus"},"claude-opus-4-5-20251101:thinking":{"cost":{"cache_read":0.5,"input":5,"output":25},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude 4.5 Opus Thinking"},"claude-opus-4-thinking":{"cost":{"cache_read":1.5,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude 4 Opus Thinking"},"claude-opus-4-thinking:1024":{"cost":{"cache_read":1.5,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude 4 Opus Thinking (1K)"},"claude-opus-4-thinking:32000":{"cost":{"cache_read":1.5,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude 4 Opus Thinking (32K)"},"claude-opus-4-thinking:32768":{"cost":{"cache_read":1.5,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude 4 Opus Thinking (32K)"},"claude-opus-4-thinking:8192":{"cost":{"cache_read":1.5,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude 4 Opus Thinking (8K)"},"claude-sonnet-4-20250514":{"cost":{"cache_read":0.3,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude 4 Sonnet"},"claude-sonnet-4-5-20250929":{"cost":{"cache_read":0.3,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude Sonnet 4.5"},"claude-sonnet-4-5-20250929-thinking":{"cost":{"cache_read":0.3,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude Sonnet 4.5 Thinking"},"claude-sonnet-4-thinking":{"cost":{"cache_read":0.3,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude 4 Sonnet Thinking"},"claude-sonnet-4-thinking:1024":{"cost":{"cache_read":0.3,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude 4 Sonnet Thinking (1K)"},"claude-sonnet-4-thinking:32768":{"cost":{"cache_read":0.3,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude 4 Sonnet Thinking (32K)"},"claude-sonnet-4-thinking:64000":{"cost":{"cache_read":0.3,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude 4 Sonnet Thinking (64K)"},"claude-sonnet-4-thinking:8192":{"cost":{"cache_read":0.3,"input":3,"output":15},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude 4 Sonnet Thinking (8K)"},"claw-high":{"cost":{"cache_read":2.5,"input":5,"output":25},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Claw High"},"claw-low":{"cost":{"cache_read":0.025,"cache_write":0.08333,"input":0.25,"output":1.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Claw Low"},"claw-medium":{"cost":{"cache_read":0.1575,"input":0.315,"output":1.26},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Claw Medium"},"cohere/command-r-plus-08-2024":{"cost":{"cache_read":1.428,"input":2.856,"output":14.246},"description":"Cohere's RAG workhorse for long-context enterprise search and tool use","name":"Cohere: Command R+"},"cohere/north-mini-code":{"cost":{"cache_read":0.1,"input":0.2,"output":0.8},"description":"Cohere coding model for practical software engineering and agentic edits","name":"Cohere North Mini Code 1.0"},"command-a-plus-05-2026":{"cost":{"cache_read":1.25,"input":2.5,"output":10},"description":"Cohere's stronger command model for multilingual agents and enterprise workflows","name":"Cohere Command A+ (05/2026)"},"command-a-reasoning-08-2025":{"cost":{"cache_read":1.25,"input":2.5,"output":10},"description":"Cohere reasoning model for multilingual enterprise agents, tools, and complex workflows","name":"Cohere Command A (08/2025)"},"crofai/greg-2-super":{"cost":{"cache_read":0.25,"input":1.5,"output":5},"description":"Greg 2 Super is CrofAI's balanced Greg 2 model for strong UI design, frontend iteration, coding, writing, and everyday agent tasks at a lower cost than Ultra.","name":"Greg 2 Super"},"crofai/greg-2-ultra":{"cost":{"cache_read":0.5,"input":3,"output":10},"description":"Greg 2 Ultra is CrofAI's most capable Greg 2 model, tuned for premium UI design, agentic coding, creative writing, and higher-end general reasoning tasks.","name":"Greg 2 Ultra"},"deepclaude":{"cost":{"cache_read":0.3,"input":3,"output":15},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"DeepClaude"},"deepcogito/cogito-v1-preview-qwen-32B":{"cost":{"cache_read":0.9,"input":1.8,"output":1.8},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Cogito v1 Preview Qwen 32B"},"deepseek-ai/DeepSeek-R1-0528":{"cost":{"cache_read":0.2,"input":0.4,"output":1.7},"description":"DeepSeek reasoning model for multi-step analysis, math, coding, and tools","name":"DeepSeek R1 0528"},"deepseek-ai/DeepSeek-V3.1":{"cost":{"cache_read":0.1,"input":0.2,"output":0.7},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek V3.1"},"deepseek-ai/DeepSeek-V3.1-Terminus":{"cost":{"cache_read":0.125,"input":0.25,"output":0.7},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek V3.1 Terminus"},"deepseek-ai/DeepSeek-V3.1-Terminus:thinking":{"cost":{"cache_read":0.125,"input":0.25,"output":0.7},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek V3.1 Terminus (Thinking)"},"deepseek-ai/DeepSeek-V3.1:thinking":{"cost":{"cache_read":0.1,"input":0.2,"output":0.7},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek V3.1 Thinking"},"deepseek-ai/deepseek-v3.2-exp":{"cost":{"cache_read":0.14,"input":0.28,"output":0.42},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek V3.2 Exp"},"deepseek-ai/deepseek-v3.2-exp-thinking":{"cost":{"cache_read":0.14,"input":0.28,"output":0.42},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek V3.2 Exp Thinking"},"deepseek-chat":{"cost":{"cache_read":0.05,"input":0.1,"output":0.425},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek V3/Deepseek Chat"},"deepseek-chat-cheaper":{"cost":{"cache_read":0.05,"input":0.1,"output":0.425},"description":"Chat-tuned GPT model for conversational assistance, writing, and tool workflows","name":"DeepSeek V3/Chat Cheaper"},"deepseek-r1":{"cost":{"cache_read":0.2,"input":0.4,"output":1.7},"description":"Classic open reasoning model for transparent math, coding, and deliberate problem solving","name":"DeepSeek R1"},"deepseek-r1-sambanova":{"cost":{"cache_read":2.499,"input":4.998,"output":6.987},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"DeepSeek R1 Fast"},"deepseek-reasoner":{"cost":{"cache_read":0.2,"input":0.4,"output":1.7},"description":"DeepSeek reasoning model for multi-step analysis, math, coding, and tools","name":"DeepSeek Reasoner"},"deepseek-reasoner-cheaper":{"cost":{"cache_read":0.2,"input":0.4,"output":1.7},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Deepseek R1 Cheaper"},"deepseek-v3-0324":{"cost":{"cache_read":0.135,"input":0.2,"output":0.77},"description":"Chat-tuned GPT model for conversational assistance, writing, and tool workflows","name":"DeepSeek Chat 0324"},"deepseek/deepseek-latest":{"cost":{"cache_read":0.11,"input":1.1,"output":2.2},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek Latest"},"deepseek/deepseek-prover-v2-671b":{"cost":{"cache_read":0.5,"input":1,"output":2.5},"description":"Flagship DeepSeek model for coding, reasoning, and agentic work","name":"DeepSeek Prover v2 671B"},"deepseek/deepseek-v3.2":{"cost":{"cache_read":0.14,"input":0.28,"output":0.42},"description":"Hybrid-reasoning DeepSeek model with thinking and non-thinking modes, sparse attention, and tool-use","name":"DeepSeek V3.2"},"deepseek/deepseek-v3.2:thinking":{"cost":{"cache_read":0.14,"input":0.28,"output":0.42},"description":"Hybrid-reasoning DeepSeek model with thinking and non-thinking modes, sparse attention, and tool-use","name":"DeepSeek V3.2 Thinking"},"deepseek/deepseek-v4-flash":{"cost":{"cache_read":0.014,"input":0.07,"output":0.14},"description":"Fast DeepSeek V4 lane for economical reasoning, coding, and long-context work","name":"DeepSeek V4 Flash"},"deepseek/deepseek-v4-flash-0731":{"cost":{"cache_read":0.014,"input":0.14,"output":0.28},"description":"Official DeepSeek V4 Flash release with enhanced agentic capabilities and integrated DSpark speculative decoding","name":"DeepSeek V4 Flash 0731"},"deepseek/deepseek-v4-flash-0731-cheaper":{"cost":{"cache_read":0.014,"input":0.14,"output":0.28},"description":"DeepSeek V4 Flash 0731 Cheaper is the same re-post-trained Mixture-of-Experts model with a 1M-token context window. This route goes directly to DeepSeek to use its lower cached-input pricing. ⚠️ Privacy and logging guarantees are limited.","name":"DeepSeek V4 Flash 0731 Cheaper"},"deepseek/deepseek-v4-flash-0731-cheaper:thinking":{"cost":{"cache_read":0.014,"input":0.14,"output":0.28},"description":"DeepSeek V4 Flash 0731 Cheaper Thinking enables reasoning by default on the same re-post-trained Mixture-of-Experts model with a 1M-token context window. This route goes directly to DeepSeek to use its lower cached-input pricing. ⚠️ Privacy and logging guarantees are limited.","name":"DeepSeek V4 Flash 0731 Cheaper (Thinking)"},"deepseek/deepseek-v4-flash-0731:thinking":{"cost":{"cache_read":0.014,"input":0.14,"output":0.28},"description":"Official DeepSeek V4 Flash release with enhanced agentic capabilities and integrated DSpark speculative decoding","name":"DeepSeek V4 Flash 0731 (Thinking)"},"deepseek/deepseek-v4-flash-latest":{"cost":{"cache_read":0.014,"input":0.14,"output":0.28},"description":"Compatibility alias that routes to the newest dated DeepSeek V4 Flash release. Currently routes to DeepSeek V4 Flash 0731. ⚠️ This route goes directly to DeepSeek, so privacy and logging guarantees are limited.","name":"DeepSeek V4 Flash Latest"},"deepseek/deepseek-v4-flash:thinking":{"cost":{"cache_read":0.014,"input":0.07,"output":0.14},"description":"Fast DeepSeek V4 lane for economical reasoning, coding, and long-context work","name":"DeepSeek V4 Flash (Thinking)"},"deepseek/deepseek-v4-pro":{"cost":{"cache_read":0.11,"input":1.1,"output":2.2},"description":"Open MoE flagship with million-token context for coding and long agent runs","name":"DeepSeek V4 Pro"},"deepseek/deepseek-v4-pro-cheaper":{"cost":{"cache_read":0.003625,"input":0.435,"output":0.87},"description":"Flagship DeepSeek model for coding, reasoning, and agentic work","name":"DeepSeek V4 Pro Cheaper"},"deepseek/deepseek-v4-pro-cheaper:thinking":{"cost":{"cache_read":0.003625,"input":0.435,"output":0.87},"description":"Flagship DeepSeek model for coding, reasoning, and agentic work","name":"DeepSeek V4 Pro Cheaper (Thinking)"},"deepseek/deepseek-v4-pro:thinking":{"cost":{"cache_read":0.11,"input":1.1,"output":2.2},"description":"Open MoE flagship with million-token context for coding and long agent runs","name":"DeepSeek V4 Pro (Thinking)"},"dmind/dmind-1-mini":{"cost":{"cache_read":0.1,"input":0.2,"output":0.4},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"DMind-1-Mini"},"doubao-1.5-pro-256k":{"cost":{"cache_read":0.3995,"input":0.799,"output":1.445},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Doubao 1.5 Pro 256k"},"doubao-1.5-pro-32k":{"cost":{"cache_read":0.06715,"input":0.1343,"output":0.3349},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Doubao 1.5 Pro 32k"},"doubao-1.5-vision-pro-32k":{"cost":{"cache_read":0.2295,"input":0.459,"output":1.377},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Doubao 1.5 Vision Pro 32k"},"doubao-seed-1-6-250615":{"cost":{"cache_read":0.102,"input":0.204,"output":0.51},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Doubao Seed 1.6"},"doubao-seed-1-6-flash-250615":{"cost":{"cache_read":0.0187,"input":0.0374,"output":0.374},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Doubao Seed 1.6 Flash"},"doubao-seed-1-8-251215":{"cost":{"cache_read":0.306,"input":0.612,"output":6.12},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Doubao Seed 1.8"},"doubao-seed-2-0-code-preview-260215":{"cost":{"cache_read":0.391,"input":0.782,"output":3.893},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Doubao Seed 2.0 Code Preview"},"doubao-seed-2-0-lite-260215":{"cost":{"cache_read":0.0731,"input":0.1462,"output":0.8738},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Doubao Seed 2.0 Lite"},"doubao-seed-2-0-mini-260215":{"cost":{"cache_read":0.02465,"input":0.0493,"output":0.4845},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Doubao Seed 2.0 Mini"},"doubao-seed-2-0-pro-260215":{"cost":{"cache_read":0.391,"input":0.782,"output":3.876},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Doubao Seed 2.0 Pro"},"ernie-5.0-thinking-preview":{"cost":{"cache_read":0.5,"input":1,"output":3.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Ernie 5.0 Thinking Preview"},"ernie-5.1":{"cost":{"cache_read":0.75,"input":0.75,"output":3},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"ERNIE 5.1"},"ernie-5.1:thinking":{"cost":{"cache_read":0.75,"input":0.75,"output":3},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"ERNIE 5.1 Thinking"},"ernie-x1.1-preview":{"cost":{"cache_read":0.075,"input":0.15,"output":0.6},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"ERNIE X1.1"},"exa-answer":{"cost":{"input":2.5,"output":2.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Exa (Answer)"},"failspy/Meta-Llama-3-70B-Instruct-abliterated-v3.5":{"cost":{"cache_read":0.35,"input":0.7,"output":0.7},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3 70B abliterated"},"fastgpt":{"cost":{"input":7.5,"output":7.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Web Answer"},"featherless-ai/Qwerky-72B":{"cost":{"cache_read":0.25,"input":0.5,"output":0.5},"description":"General-purpose chat model for instruction following, writing, and analysis","name":"Qwerky 72B"},"gemini-2.0-pro-exp-02-05":{"cost":{"cache_read":0.49725,"input":1.989,"output":7.956},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.0 Pro 0205"},"gemini-2.0-pro-reasoner":{"cost":{"cache_read":0.323,"input":1.292,"output":4.998},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.0 Pro Reasoner"},"gemini-2.5-flash":{"cost":{"cache_read":0.03,"input":0.3,"output":2.5},"description":"Fast Gemini workhorse for multimodal apps where latency and price matter","name":"Gemini 2.5 Flash"},"gemini-2.5-flash-lite":{"cost":{"cache_read":0.01,"input":0.1,"output":0.4},"description":"Lean Gemini 2.5 lane for cheap multimodal traffic and quick agents","name":"Gemini 2.5 Flash Lite"},"gemini-2.5-flash-lite-preview-06-17":{"cost":{"cache_read":0.015,"input":0.15,"output":0.6},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.5 Flash Lite Preview"},"gemini-2.5-flash-lite-preview-09-2025":{"cost":{"cache_read":0.01,"input":0.1,"output":0.4},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.5 Flash Lite Preview (09/2025)"},"gemini-2.5-flash-lite-preview-09-2025-thinking":{"cost":{"cache_read":0.01,"input":0.1,"output":0.4},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.5 Flash Lite Preview (09/2025) – Thinking"},"gemini-2.5-flash-nothinking":{"cost":{"cache_read":0.03,"input":0.3,"output":2.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.5 Flash (No Thinking)"},"gemini-2.5-flash-preview-04-17":{"cost":{"cache_read":0.015,"input":0.15,"output":0.6},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.5 Flash Preview"},"gemini-2.5-flash-preview-04-17:thinking":{"cost":{"cache_read":0.015,"input":0.15,"output":3.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.5 Flash Preview Thinking"},"gemini-2.5-flash-preview-05-20":{"cost":{"cache_read":0.015,"input":0.15,"output":0.6},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.5 Flash 0520"},"gemini-2.5-flash-preview-05-20:thinking":{"cost":{"cache_read":0.015,"input":0.15,"output":3.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.5 Flash 0520 Thinking"},"gemini-2.5-flash-preview-09-2025":{"cost":{"cache_read":0.03,"input":0.3,"output":2.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.5 Flash Preview (09/2025)"},"gemini-2.5-flash-preview-09-2025-thinking":{"cost":{"cache_read":0.03,"input":0.3,"output":2.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.5 Flash Preview (09/2025) – Thinking"},"gemini-2.5-pro":{"cost":{"cache_read":0.125,"cache_write":0.375,"input":1.25,"output":10},"description":"Google's proven reasoning model for coding, math, and multimodal analysis","name":"Gemini 2.5 Pro"},"gemini-2.5-pro-exp-03-25":{"cost":{"cache_read":0.25,"input":2.5,"output":10},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.5 Pro Experimental 0325"},"gemini-2.5-pro-preview-03-25":{"cost":{"cache_read":0.25,"input":2.5,"output":10},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.5 Pro Preview 0325"},"gemini-2.5-pro-preview-05-06":{"cost":{"cache_read":0.25,"input":2.5,"output":10},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.5 Pro Preview 0506"},"gemini-2.5-pro-preview-06-05":{"cost":{"cache_read":0.25,"input":2.5,"output":10},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.5 Pro Preview 0605"},"gemini-3-pro-image-preview":{"cost":{"cache_read":0.2,"input":2,"output":12},"description":"Nano Banana Pro for higher-fidelity image generation and design-heavy edits","name":"Gemini 3 Pro Image"},"gemini-exp-1206":{"cost":{"cache_read":0.629,"input":1.258,"output":4.998},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini 2.0 Pro 1206"},"gemma-4-12b-it":{"cost":{"cache_read":0.03,"input":0.06,"output":0.3},"description":"Google's Gemma 4 12B Instruct is an open-weight multimodal model for text, image, audio, and video understanding, with tool calling and structured output support.","name":"Gemma 4 12B Instruct"},"gemma-4-e2b-it":{"cost":{"cache_read":0.01,"input":0.02,"output":0.1},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 4 E2B Instruct"},"gemma-4-e4b-it":{"cost":{"cache_read":0.02,"input":0.04,"output":0.2},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 4 E4B Instruct"},"glm-4":{"cost":{"cache_read":7.497,"input":14.994,"output":14.994},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GLM-4"},"glm-4-air":{"cost":{"cache_read":0.1003,"input":0.2006,"output":0.2006},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GLM-4 Air"},"glm-4-air-0111":{"cost":{"cache_read":0.0697,"input":0.1394,"output":0.1394},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GLM 4 Air 0111"},"glm-4-airx":{"cost":{"cache_read":1.003,"input":2.006,"output":2.006},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GLM-4 AirX"},"glm-4-flash":{"cost":{"cache_read":0.05015,"input":0.1003,"output":0.1003},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GLM-4 Flash"},"glm-4-long":{"cost":{"cache_read":0.1003,"input":0.2006,"output":0.2006},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GLM-4 Long"},"glm-4-plus":{"cost":{"cache_read":3.7485,"input":7.497,"output":7.497},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GLM-4 Plus"},"glm-4-plus-0111":{"cost":{"cache_read":4.998,"input":9.996,"output":9.996},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GLM 4 Plus 0111"},"glm-4.1v-thinking-flash":{"cost":{"cache_read":0.15,"input":0.3,"output":0.3},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GLM 4.1V Thinking Flash"},"glm-4.1v-thinking-flashx":{"cost":{"cache_read":0.15,"input":0.3,"output":0.3},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GLM 4.1V Thinking FlashX"},"glm-z1-air":{"cost":{"cache_read":0.035,"input":0.07,"output":0.07},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GLM Z1 Air"},"glm-z1-airx":{"cost":{"cache_read":0.35,"input":0.7,"output":0.7},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GLM Z1 AirX"},"glm-zero-preview":{"cost":{"cache_read":0.901,"input":1.802,"output":1.802},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GLM Zero Preview"},"google/gemini-3-flash-preview":{"cost":{"cache_read":0.05,"input":0.5,"output":3},"description":"New Gemini flash lane bringing frontier-style multimodal reasoning to cheaper runs","name":"Gemini 3 Flash (Preview)"},"google/gemini-3-flash-preview-thinking":{"cost":{"cache_read":0.05,"input":0.5,"output":3},"description":"New Gemini flash lane bringing frontier-style multimodal reasoning to cheaper runs","name":"Gemini 3 Flash Thinking"},"google/gemini-3.1-flash-lite":{"cost":{"cache_read":0.025,"cache_write":0.08333,"input":0.25,"output":1.5},"description":"Low-latency Gemini model for high-volume multimodal and agent workloads","name":"Gemini 3.1 Flash Lite"},"google/gemini-3.1-pro-preview":{"cost":{"cache_read":0.2,"cache_write":0.375,"input":2,"output":12},"description":"Reasoning-first Gemini preview for agentic coding and complex problem solving","name":"Gemini 3.1 Pro (Preview)"},"google/gemini-3.1-pro-preview-customtools":{"cost":{"cache_read":0.2,"cache_write":0.375,"input":2,"output":12},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Gemini 3.1 Pro (Preview Custom Tools)"},"google/gemini-3.1-pro-preview-high":{"cost":{"cache_read":0.2,"cache_write":0.375,"input":2,"output":12},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Gemini 3.1 Pro (Preview High)"},"google/gemini-3.1-pro-preview-low":{"cost":{"cache_read":0.2,"cache_write":0.375,"input":2,"output":12},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Gemini 3.1 Pro (Preview Low)"},"google/gemini-3.5-flash":{"cost":{"cache_read":0.15,"input":1.5,"output":9},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini 3.5 Flash"},"google/gemini-3.5-flash-lite":{"cost":{"cache_read":0.03,"cache_write":0.08333,"input":0.3,"output":2.5},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini 3.5 Flash Lite"},"google/gemini-3.5-flash-thinking":{"cost":{"cache_read":0.15,"input":1.5,"output":9},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini 3.5 Flash Thinking"},"google/gemini-3.6-flash":{"cost":{"cache_read":0.15,"input":1.5,"output":7.5},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini 3.6 Flash"},"google/gemini-flash-latest":{"cost":{"cache_read":0.15,"input":1.5,"output":7.5},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini Flash Latest"},"google/gemini-flash-lite-latest":{"cost":{"cache_read":0.03,"cache_write":0.08333,"input":0.3,"output":2.5},"description":"Low-latency Gemini model for high-volume multimodal and agent workloads","name":"Gemini Flash Lite Latest"},"google/gemini-pro-latest":{"cost":{"cache_read":0.2,"cache_write":0.375,"input":2,"output":12},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Gemini Pro Latest"},"google/gemma-4-26b-a4b-it":{"cost":{"cache_read":0.065,"input":0.13,"output":0.4},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 4 26B A4B"},"google/gemma-4-26b-a4b-it:thinking":{"cost":{"cache_read":0.065,"input":0.13,"output":0.4},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 4 26B A4B Thinking"},"google/gemma-4-31b-it":{"cost":{"cache_read":0.05,"input":0.1,"output":0.35},"description":"Largest Gemma 4 instruction model for open, self-hosted chat and reasoning","name":"Gemma 4 31B"},"google/gemma-4-31b-it:thinking":{"cost":{"cache_read":0.05,"input":0.1,"output":0.35},"description":"Largest Gemma 4 instruction model for open, self-hosted chat and reasoning","name":"Gemma 4 31B Thinking"},"hermes-high":{"cost":{"cache_read":2.5,"input":5,"output":25},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Hermes High"},"hermes-low":{"cost":{"cache_read":0.025,"cache_write":0.08333,"input":0.25,"output":1.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Hermes Low"},"hermes-medium":{"cost":{"cache_read":0.1575,"input":0.315,"output":1.26},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Hermes Medium"},"holo3-35b-a3b":{"cost":{"cache_read":0.125,"input":0.25,"output":1.8},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Holo3-35B-A3B"},"holo3-35b-a3b:thinking":{"cost":{"cache_read":0.125,"input":0.25,"output":1.8},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Holo3-35B-A3B Thinking"},"huihui-ai/DeepSeek-R1-Distill-Llama-70B-abliterated":{"cost":{"cache_read":0.35,"input":0.7,"output":0.7},"description":"DeepSeek reasoning model for multi-step analysis, math, coding, and tools","name":"DeepSeek R1 Llama 70B Abliterated"},"huihui-ai/DeepSeek-R1-Distill-Qwen-32B-abliterated":{"cost":{"cache_read":0.7,"input":1.4,"output":1.4},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"DeepSeek R1 Qwen Abliterated"},"huihui-ai/Llama-3.3-70B-Instruct-abliterated":{"cost":{"cache_read":0.35,"input":0.7,"output":0.7},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.3 70B Instruct abliterated"},"huihui-ai/Qwen2.5-32B-Instruct-abliterated":{"cost":{"cache_read":0.35,"input":0.7,"output":0.7},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen 2.5 32B Abliterated"},"hunyuan-turbos-20250226":{"cost":{"cache_read":0.0935,"input":0.187,"output":0.374},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Hunyuan Turbo S"},"ibm-granite/granite-4.1-8b":{"cost":{"cache_read":0.05,"input":0.05,"output":0.1},"description":"Tool-capable chat model for instruction following and agentic application workflows","name":"Granite 4.1 8B"},"inclusionai/ling-2.6-1t":{"cost":{"cache_read":0.06,"input":0.3,"output":2.5},"description":"Tool-capable chat model for instruction following and agentic application workflows","name":"Ling 2.6 1T"},"inclusionai/ling-2.6-flash":{"cost":{"cache_read":0.02,"input":0.1,"output":0.3},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Ling 2.6 Flash"},"inclusionai/ling-3.0-flash":{"cost":{"cache_read":0.015,"input":0.075,"output":0.22},"description":"Ling-3.0-flash is a 124B-parameter Mixture-of-Experts model with approximately 5.1B parameters active per token. It prioritizes token efficiency and production-scale agentic inference, helping coding and tool-using agents complete more work within constrained latency and serving budgets.","name":"Ling 3.0 Flash"},"inclusionai/ling-3.0-flash:thinking":{"cost":{"cache_read":0.015,"input":0.075,"output":0.22},"description":"Ling-3.0-flash Thinking enables visible reasoning on inclusionAI's token-efficient 124B-parameter Mixture-of-Experts model for harder coding, tool use, planning, and production-scale agent workflows.","name":"Ling 3.0 Flash Thinking"},"inclusionai/ring-2.6-1t":{"cost":{"cache_read":0.06,"input":0.3,"output":2.5},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"Ring 2.6 1T"},"inflatebot/MN-12B-Mag-Mell-R1":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mag Mell R1"},"inflection/inflection-3-pi":{"cost":{"cache_read":1.2495,"input":2.499,"output":9.996},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"Inflection 3 Pi"},"inflection/inflection-3-productivity":{"cost":{"cache_read":1.2495,"input":2.499,"output":9.996},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"Inflection 3 Productivity"},"jamba-large":{"cost":{"cache_read":0.9945,"input":1.989,"output":7.99},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Jamba Large"},"jamba-large-1.6":{"cost":{"cache_read":0.9945,"input":1.989,"output":7.99},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Jamba Large 1.6"},"jamba-large-1.7":{"cost":{"cache_read":0.9945,"input":1.989,"output":7.99},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Jamba Large 1.7"},"jamba-mini":{"cost":{"cache_read":0.09945,"input":0.1989,"output":0.408},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Jamba Mini"},"jamba-mini-1.6":{"cost":{"cache_read":0.09945,"input":0.1989,"output":0.408},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Jamba Mini 1.6"},"jamba-mini-1.7":{"cost":{"cache_read":0.09945,"input":0.1989,"output":0.408},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Jamba Mini 1.7"},"kimi-k2-instruct-fast":{"cost":{"cache_read":0.2,"input":0.4,"output":1.8},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Kimi K2 0711 Fast"},"kwaipilot/kat-coder-air-v2.5":{"cost":{"cache_read":0.03,"input":0.15,"output":0.6},"description":"Fast, cost-efficient KAT Coder model for code generation, editing, debugging, and agentic software-development workflows.","name":"KAT Coder Air V2.5"},"kwaipilot/kat-coder-pro-v2":{"cost":{"cache_read":0.15,"input":0.3,"output":1.2},"description":"Coding model for repository understanding, refactors, and agentic engineering tasks","name":"KAT Coder Pro V2"},"kwaipilot/kat-coder-pro-v2.5":{"cost":{"cache_read":0.15,"input":0.74,"output":2.96},"description":"Higher-capability KAT Coder model for complex code generation, repository-scale editing, debugging, and agentic software-development workflows.","name":"KAT Coder Pro V2.5"},"learnlm-1.5-pro-experimental":{"cost":{"cache_read":1.751,"input":3.502,"output":10.506},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Gemini LearnLM Experimental"},"longcat-2.0":{"cost":{"cache_read":0.015,"input":0.75,"output":3},"description":"Meituan LongCat-2.0, a reasoning model with tool calling and a 1M-token context window","name":"LongCat 2.0"},"longcat-2.0:thinking":{"cost":{"cache_read":0.015,"input":0.75,"output":3},"description":"Meituan LongCat-2.0, a reasoning model with tool calling and a 1M-token context window","name":"LongCat 2.0 Thinking"},"meganova-ai/manta-flash-1.0":{"cost":{"cache_read":0.01,"input":0.02,"output":0.16},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Manta Flash 1.0"},"meganova-ai/manta-mini-1.0":{"cost":{"cache_read":0.01,"input":0.02,"output":0.16},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Manta Mini 1.0"},"meganova-ai/manta-pro-1.0":{"cost":{"cache_read":0.03,"input":0.06,"output":0.5},"description":"Flagship model for demanding analysis, coding, and production agent workflows","name":"Manta Pro 1.0"},"mercury-2":{"cost":{"cache_read":0.025,"input":0.25,"output":0.75},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Mercury 2"},"mercury-coder-small":{"cost":{"cache_read":0.125,"input":0.25,"output":1},"description":"Model by Inception AI. A diffusion large language model that runs incredibly quickly (500+ tokens/second) while matching Claude 3.5 Haiku and GPT-4o-mini. 1st in speed on Copilot arena, and matching 2nd in quality.","name":"Mercury Coder Small"},"meta-llama/llama-3.1-8b-instruct":{"cost":{"cache_read":0.0272,"input":0.0544,"output":0.085},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.1 8b Instruct"},"meta-llama/llama-3.2-3b-instruct":{"cost":{"cache_read":0.0153,"input":0.0306,"output":0.0493},"description":"Open Llama multimodal model for image understanding and text reasoning","name":"Llama 3.2 3b Instruct"},"meta-llama/llama-3.3-70b-instruct":{"cost":{"cache_read":0.025,"input":0.05,"output":0.23},"description":"Popular open Llama workhorse for multilingual chat, coding, and self-hosting","name":"Llama 3.3 70b Instruct"},"meta-llama/llama-4-maverick":{"cost":{"cache_read":0.075,"input":0.15,"output":0.6},"description":"Open multimodal Llama model for strong reasoning and fast responses","name":"Llama 4 Maverick"},"meta-llama/llama-4-scout":{"cost":{"cache_read":0.0425,"input":0.085,"output":0.46},"description":"Open multimodal Llama model for long-context analysis and efficient agents","name":"Llama 4 Scout"},"meta/muse-spark-1.1":{"cost":{"cache_read":0.15,"input":1.25,"output":4.25},"description":"Muse Spark is a natively multimodal reasoning model with support for tool-use, visual chain of thought, and multi-agent orchestration.","name":"Muse Spark 1.1"},"meta/muse-spark-1.2":{"cost":{"cache_read":0.15,"input":1.25,"output":4.25},"description":"Muse Spark 1.2 is a coding-focused update to Muse Spark 1.1 with improvements in code generation, complex debugging, codebase understanding, and end-to-end developer workflows.","name":"Muse Spark 1.2"},"meta/muse-spark-1.2-contributor":{"cost":{"cache_read":0.002,"input":0.1,"output":0.2},"description":"A much cheaper opt-in version of Muse Spark 1.2 with the same multimodal coding and agentic capabilities. Prompts and outputs sent to this Contributor model may be used by Meta for training and to improve its products; use the standard Muse Spark 1.2 model if you do not want your data used for training.","name":"Muse Spark 1.2 Contributor (Data Used for Training)"},"microsoft/wizardlm-2-8x22b":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"WizardLM-2 8x22B"},"minimax/minimax-01":{"cost":{"cache_read":0.0697,"input":0.1394,"output":1.122},"description":"MiniMax multimodal coding model for long-context reasoning and agent tasks","name":"MiniMax 01"},"minimax/minimax-latest":{"cost":{"cache_read":0.06,"input":0.3,"output":1.2},"description":"MiniMax multimodal coding model for long-context reasoning and agent tasks","name":"MiniMax Latest"},"minimax/minimax-m2-her":{"cost":{"cache_read":0.151,"input":0.302,"output":1.207},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"MiniMax M2-her"},"minimax/minimax-m2.1":{"cost":{"cache_read":0.165,"input":0.33,"output":1.32},"description":"Earlier MiniMax agent model for practical coding and productivity tasks","name":"MiniMax M2.1"},"minimax/minimax-m2.5":{"cost":{"cache_read":0.15,"input":0.3,"output":1.2},"description":"Prior MiniMax coding model for agent workflows, office edits, and automation","name":"MiniMax M2.5"},"minimax/minimax-m2.7":{"cost":{"cache_read":0.1575,"input":0.315,"output":1.26},"description":"Open MiniMax flagship for coding agents, office automation, and complex environments","name":"MiniMax M2.7"},"minimax/minimax-m2.7-turbo":{"cost":{"cache_read":0.3,"input":0.6,"output":2.4},"description":"Efficient MiniMax model for quick assistance, coding, and routine automation","name":"MiniMax M2.7 Turbo"},"minimax/minimax-m3":{"cost":{"cache_read":0.06,"input":0.3,"output":1.2},"description":"MiniMax multimodal model for long-context coding, perception, and agent planning","name":"MiniMax M3"},"minimax/minimax-m3:thinking":{"cost":{"cache_read":0.06,"input":0.3,"output":1.2},"description":"MiniMax multimodal model for long-context coding, perception, and agent planning","name":"MiniMax M3 Thinking"},"mirothinker-1-7-deepresearch":{"cost":{"cache_read":2,"input":4,"output":25},"description":"Research model for long-horizon investigation, synthesis, and analytical reports","name":"MiroThinker 1.7 Deep Research"},"mirothinker-1-7-deepresearch-mini":{"cost":{"cache_read":0.625,"input":1.25,"output":10},"description":"Research model for long-horizon investigation, synthesis, and analytical reports","name":"MiroThinker 1.7 Deep Research Mini"},"mistral-code-agent-latest":{"cost":{"cache_read":0.2,"input":0.4,"output":2},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Mistral Code Agent Latest"},"mistral-code-latest":{"cost":{"cache_read":0.15,"input":0.3,"output":0.9},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Mistral Code Latest"},"mistral-small-31-24b-instruct":{"cost":{"cache_read":0.05,"input":0.1,"output":0.3},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Mistral Small 31 24b Instruct"},"mistral/mistral-medium-3.5":{"cost":{"cache_read":0.75,"input":1.5,"output":7.5},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral Medium 3.5"},"mistral/mistral-medium-3.5:thinking":{"cost":{"cache_read":0.75,"input":1.5,"output":7.5},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral Medium 3.5 Thinking"},"mistralai/Devstral-Small-2505":{"cost":{"cache_read":0.03,"input":0.06,"output":0.06},"description":"Mistral coding agent model for repository tasks and software engineering workflows","name":"Mistral Devstral Small 2505"},"mistralai/Mistral-Nemo-Instruct-2407":{"cost":{"cache_read":0.05015,"input":0.1003,"output":0.1207},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral Nemo"},"mistralai/codestral-2508":{"cost":{"cache_read":0.15,"input":0.3,"output":0.9},"description":"Mistral coding model for code completion, generation, and developer workflows","name":"Codestral 2508"},"mistralai/devstral-2-123b-instruct-2512":{"cost":{"cache_read":0.2,"input":0.4,"output":1.4},"description":"Mistral coding agent model for repository tasks and software engineering workflows","name":"Devstral 2 123B"},"mistralai/ministral-14b-2512":{"cost":{"cache_read":0.1,"input":0.2,"output":0.2},"description":"Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads","name":"Ministral 14B"},"mistralai/ministral-14b-instruct-2512":{"cost":{"cache_read":0.05,"input":0.1,"output":0.4},"description":"Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads","name":"Ministral 3 14B"},"mistralai/ministral-3b-2512":{"cost":{"cache_read":0.05,"input":0.1,"output":0.1},"description":"Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads","name":"Ministral 3B"},"mistralai/ministral-8b-2512":{"cost":{"cache_read":0.075,"input":0.15,"output":0.15},"description":"Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads","name":"Ministral 8B"},"mistralai/mistral-large":{"cost":{"cache_read":0.2,"input":2.006,"output":6.001},"description":"Flagship Mistral model for advanced reasoning, coding, and multilingual work","name":"Mistral Large 2411"},"mistralai/mistral-large-3-675b-instruct-2512":{"cost":{"cache_read":0.5,"input":1,"output":3},"description":"Flagship Mistral model for advanced reasoning, coding, and multilingual work","name":"Mistral Large 3 675B"},"mistralai/mistral-medium-3":{"cost":{"cache_read":0.2,"input":0.4,"output":2},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral Medium 3"},"mistralai/mistral-medium-3.1":{"cost":{"cache_read":0.2,"input":0.4,"output":2},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral Medium 3.1"},"mistralai/mistral-saba":{"cost":{"cache_read":0.09945,"input":0.1989,"output":0.595},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral Saba"},"mistralai/mistral-small-4-119b-2603":{"cost":{"cache_read":0.2,"input":0.4,"output":1.4},"description":"Efficient Mistral model for fast chat, extraction, and production assistants","name":"Mistral Small 4 119B"},"mistralai/mistral-small-4-119b-2603:thinking":{"cost":{"cache_read":0.2,"input":0.4,"output":1.4},"description":"Efficient Mistral model for fast chat, extraction, and production assistants","name":"Mistral Small 4 119B Thinking"},"mistralai/mixtral-8x22b-instruct-v0.1":{"cost":{"cache_read":0.2,"input":2,"output":6},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mixtral 8x22B"},"mlabonne/NeuralDaredevil-8B-abliterated":{"cost":{"cache_read":0.22,"input":0.44,"output":0.44},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Neural Daredevil 8B abliterated"},"moonshotai/Kimi-K2-Instruct-0905":{"cost":{"cache_read":0.2,"input":0.4,"output":1.8},"description":"Kimi model for long-context chat, coding, and agentic reasoning","name":"Kimi K2 0905"},"moonshotai/kimi-k2-instruct":{"cost":{"cache_read":0.2,"input":0.4,"output":1.8},"description":"Kimi model for long-context chat, coding, and agentic reasoning","name":"Kimi K2 Instruct"},"moonshotai/kimi-k2-instruct-0711":{"cost":{"cache_read":0.2,"input":0.4,"output":1.8},"description":"Kimi model for long-context chat, coding, and agentic reasoning","name":"Kimi K2 0711"},"moonshotai/kimi-k2-thinking":{"cost":{"cache_read":0.15,"input":0.6,"output":2.5},"description":"Thinking Kimi model for slower research passes, planning, and hard technical questions","name":"Kimi K2 Thinking"},"moonshotai/kimi-k2.5":{"cost":{"cache_read":0.15,"input":0.3,"output":1.9},"description":"Earlier Kimi frontier model for long-context agents, coding, and multimodal work","name":"Kimi K2.5"},"moonshotai/kimi-k2.5:thinking":{"cost":{"cache_read":0.15,"input":0.3,"output":1.9},"description":"Earlier Kimi frontier model for long-context agents, coding, and multimodal work","name":"Kimi K2.5 Thinking"},"moonshotai/kimi-k2.6":{"cost":{"cache_read":0.125,"input":0.5,"output":2.6},"description":"Multimodal Kimi workhorse for agent loops, coding tasks, and visual context","name":"Kimi K2.6"},"moonshotai/kimi-k2.6:thinking":{"cost":{"cache_read":0.125,"input":0.5,"output":2.6},"description":"Multimodal Kimi workhorse for agent loops, coding tasks, and visual context","name":"Kimi K2.6 Thinking"},"moonshotai/kimi-k2.7-code":{"cost":{"cache_read":0.19,"input":0.95,"output":4},"description":"Coding-focused Kimi model, stronger on long-horizon repo work with less overthinking","name":"Kimi K2.7 Code"},"moonshotai/kimi-k2.7-code-highspeed":{"cost":{"cache_read":0.32,"input":1.9,"output":8},"description":"Lower-latency Kimi Code variant for interactive edits and coding-agent loops","name":"Kimi K2.7 Code High-Speed"},"moonshotai/kimi-k3":{"cost":{"cache_read":0.25,"input":2.5,"output":13.5},"description":"Multimodal Kimi model with 1M context and toggleable max-effort thinking for long-horizon agent work","name":"Kimi K3"},"moonshotai/kimi-latest":{"cost":{"cache_read":0.25,"input":2.5,"output":13.5},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi Latest"},"nano-gpt-help":{"cost":{"input":0,"output":0},"description":"Text-only NanoGPT support assistant. Questions are processed by the Help inference provider; do not paste secrets or account credentials. Covers the website, models, API, pricing, memory, media generation, and support.","name":"NanoGPT Help"},"nanogpt/coding-router":{"cost":{"cache_read":0.11,"input":1.1,"output":2.2},"description":"Automatic model router for matching prompts to suitable backends and budgets","name":"Coding Router"},"nanogpt/coding-router:high":{"cost":{"cache_read":0.11,"input":1.1,"output":2.2},"description":"Automatic model router for matching prompts to suitable backends and budgets","name":"Coding Router High"},"nanogpt/coding-router:low":{"cost":{"cache_read":0.028,"input":0.14,"output":0.28},"description":"Automatic model router for matching prompts to suitable backends and budgets","name":"Coding Router Low"},"nanogpt/coding-router:max":{"cost":{"cache_read":0.5,"input":5,"output":30},"description":"Automatic model router for matching prompts to suitable backends and budgets","name":"Coding Router Max"},"nanogpt/coding-router:medium":{"cost":{"cache_read":0.028,"input":0.14,"output":0.28},"description":"Automatic model router for matching prompts to suitable backends and budgets","name":"Coding Router Medium"},"nex-agi/nex-n2-mini":{"cost":{"cache_read":0.0025,"input":0.025,"output":0.1},"description":"Nex AGI's open-source agentic mixture-of-experts model in the Nex N2 family. It accepts text and image input and is built for coding, tool use, structured outputs, and optional reasoning with a 256K context window.","name":"Nex N2 Mini"},"nex-agi/nex-n2-pro":{"cost":{"cache_read":0.25,"input":0.5,"output":2.5},"description":"Nex AGI's open-source agentic reasoning model, post-trained on Qwen3.5-397B-A17B. It is built for agentic coding, software engineering, deep research, tool use, and long-horizon tasks with a 256K context window.","name":"Nex N2 Pro"},"nothingiisreal/L3.1-70B-Celeste-V0.1-BF16":{"cost":{"cache_read":0.2465,"input":0.493,"output":0.493},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.1 70B Celeste v0.1"},"nvidia/Llama-3.1-Nemotron-70B-Instruct-HF":{"cost":{"cache_read":0.1785,"input":0.357,"output":0.408},"description":"Nemotron model for efficient reasoning, coding, and specialized AI agents","name":"Nvidia Nemotron 70b"},"nvidia/Llama-3.3-Nemotron-Super-49B-v1":{"cost":{"cache_read":0.075,"input":0.15,"output":0.15},"description":"Nemotron model for efficient reasoning, coding, and specialized AI agents","name":"Nvidia Nemotron Super 49B"},"nvidia/nemotron-3-nano-30b-a3b":{"cost":{"cache_read":0.085,"input":0.17,"output":0.68},"description":"Small Nemotron 3 MoE for efficient coding, math, and long-context agents","name":"Nvidia Nemotron 3 Nano 30B"},"nvidia/nemotron-3-nano-omni-30b-a3b-reasoning":{"cost":{"cache_read":0.0525,"input":0.105,"output":0.42},"description":"Open Nemotron omni model combining reasoning with text, vision, and audio","name":"Nvidia Nemotron 3 Nano Omni"},"nvidia/nemotron-3-super-120b-a12b":{"cost":{"cache_read":0.025,"input":0.05,"output":0.25},"description":"Nemotron middle tier for collaborative agents and high-volume reasoning workloads","name":"Nvidia Nemotron 3 Super 120B"},"nvidia/nemotron-3-super-120b-a12b:thinking":{"cost":{"cache_read":0.025,"input":0.05,"output":0.25},"description":"Nemotron middle tier for collaborative agents and high-volume reasoning workloads","name":"Nvidia Nemotron 3 Super 120B Thinking"},"nvidia/nemotron-3-ultra-550b-a55b":{"cost":{"cache_read":0.25,"input":0.5,"output":2.5},"description":"Largest Nemotron 3 model for maximum open-weight reasoning and agent accuracy","name":"Nvidia Nemotron 3 Ultra 550B"},"nvidia/nemotron-3-ultra-550b-a55b:thinking":{"cost":{"cache_read":0.25,"input":0.5,"output":2.5},"description":"Largest Nemotron 3 model for maximum open-weight reasoning and agent accuracy","name":"Nvidia Nemotron 3 Ultra 550B Thinking"},"openai/gpt-3.5-turbo":{"cost":{"input":0.5,"output":1.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-3.5 Turbo"},"openai/gpt-4-turbo":{"cost":{"input":10,"output":30},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-4 Turbo"},"openai/gpt-4-turbo-preview":{"cost":{"input":10,"output":30},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-4 Turbo Preview"},"openai/gpt-4.1":{"cost":{"cache_read":0.5,"input":2,"output":8},"description":"Long-lived GPT workhorse for coding, instruction following, and production apps","name":"GPT 4.1"},"openai/gpt-4.1-mini":{"cost":{"cache_read":0.1,"input":0.4,"output":1.6},"description":"Affordable GPT-4.1 lane for fast coding help and structured extraction","name":"GPT 4.1 Mini"},"openai/gpt-4.1-nano":{"cost":{"cache_read":0.025,"input":0.1,"output":0.4},"description":"Tiny GPT-4.1 option for classification, routing, and very high-volume tasks","name":"GPT 4.1 Nano"},"openai/gpt-4o":{"cost":{"cache_read":1.25,"input":2.5,"output":10},"description":"Omni-era GPT for multimodal chat, practical coding, and general assistants","name":"GPT-4o"},"openai/gpt-4o-2024-08-06":{"cost":{"cache_read":1.25,"input":2.5,"output":10},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-4o (2024-08-06)"},"openai/gpt-4o-2024-11-20":{"cost":{"cache_read":1.25,"input":2.5,"output":10},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-4o (2024-11-20)"},"openai/gpt-4o-mini":{"cost":{"cache_read":0.075,"input":0.15,"output":0.6},"description":"Small omni GPT for cheap multimodal assistance and production-scale traffic","name":"GPT-4o mini"},"openai/gpt-4o-mini-search-preview":{"cost":{"cache_read":0.075,"input":0.15,"output":0.6},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-4o mini Search Preview"},"openai/gpt-4o-search-preview":{"cost":{"cache_read":1.25,"input":2.5,"output":10},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-4o Search Preview"},"openai/gpt-5":{"cost":{"cache_read":0.125,"input":1.25,"output":10},"description":"Original GPT-5 workhorse for reasoning, coding, writing, and tool workflows","name":"GPT 5"},"openai/gpt-5-codex":{"cost":{"cache_read":0.125,"input":1.25,"output":10},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5 Codex"},"openai/gpt-5-mini":{"cost":{"cache_read":0.025,"input":0.25,"output":2},"description":"Small GPT-5 for responsive agents, coding help, and everyday automation","name":"GPT 5 Mini"},"openai/gpt-5-nano":{"cost":{"cache_read":0.005,"input":0.05,"output":0.4},"description":"Tiny GPT-5 lane for routing, extraction, classification, and bulk jobs","name":"GPT 5 Nano"},"openai/gpt-5-pro":{"cost":{"cache_read":1.5,"input":15,"output":120},"description":"Higher-accuracy GPT-5 tier for tough analysis, coding reviews, and planning","name":"GPT 5 Pro"},"openai/gpt-5.1":{"cost":{"cache_read":0.125,"input":1.25,"output":10},"description":"Sharper GPT-5 generation for coding, product work, and tool-assisted tasks","name":"GPT 5.1"},"openai/gpt-5.1-2025-11-13":{"cost":{"cache_read":0.125,"input":1.25,"output":10},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-5.1 (2025-11-13)"},"openai/gpt-5.1-codex":{"cost":{"cache_read":0.125,"input":1.25,"output":10},"description":"Codex GPT for repository edits, code review, and practical software agents","name":"GPT 5.1 Codex"},"openai/gpt-5.1-codex-max":{"cost":{"cache_read":0.25,"input":2.5,"output":20},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT 5.1 Codex Max"},"openai/gpt-5.1-codex-mini":{"cost":{"cache_read":0.025,"input":0.25,"output":2},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT 5.1 Codex Mini"},"openai/gpt-5.2":{"cost":{"cache_read":0.175,"input":1.75,"output":14},"description":"Reliable GPT generation for broad coding, writing, and tool-assisted product work","name":"GPT 5.2"},"openai/gpt-5.2-codex":{"cost":{"cache_read":0.175,"input":1.75,"output":14},"description":"Code-specialist GPT for repository edits, reviews, and long-running software agents","name":"GPT 5.2 Codex"},"openai/gpt-5.3-codex":{"cost":{"cache_read":0.175,"input":1.75,"output":14},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT 5.3 Codex"},"openai/gpt-5.4":{"cost":{"cache_read":0.25,"input":2.5,"output":15},"description":"Agent-ready GPT for coding and computer-use workflows at a lower cost","name":"GPT 5.4"},"openai/gpt-5.4-mini":{"cost":{"cache_read":0.075,"input":0.75,"output":4.5},"description":"Strong small GPT for coding subagents, quick tool use, and high-volume work","name":"GPT 5.4 Mini"},"openai/gpt-5.4-nano":{"cost":{"cache_read":0.02,"input":0.2,"output":1.25},"description":"Cheapest GPT-5.4 lane for simple routing, extraction, and bulk automation","name":"GPT 5.4 Nano"},"openai/gpt-5.5":{"cost":{"cache_read":0.5,"input":5,"output":30},"description":"Default frontier GPT for coding, computer use, research, and knowledge work","name":"GPT 5.5"},"openai/gpt-5.6-luna":{"cost":{"cache_read":0.01,"cache_write":0.125,"input":0.1,"output":0.6},"description":"Cost-efficient GPT-5.6 model for fast, high-volume workloads","name":"GPT 5.6 Luna"},"openai/gpt-5.6-luna-pro":{"cost":{"cache_read":0.01,"cache_write":0.125,"input":0.1,"output":0.6},"description":"Cost-efficient GPT-5.6 model for fast, high-volume workloads","name":"GPT 5.6 Luna Pro"},"openai/gpt-5.6-sol":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":30},"description":"Frontier GPT-5.6 model for complex professional work, coding, and agentic workflows","name":"GPT 5.6 Sol"},"openai/gpt-5.6-sol-pro":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":30},"description":"Frontier GPT-5.6 model for complex professional work, coding, and agentic workflows","name":"GPT 5.6 Sol Pro"},"openai/gpt-5.6-terra":{"cost":{"cache_read":0.1,"cache_write":1.25,"input":1,"output":6},"description":"Balanced GPT-5.6 model for capable, cost-efficient everyday work","name":"GPT 5.6 Terra"},"openai/gpt-5.6-terra-pro":{"cost":{"cache_read":0.1,"cache_write":1.25,"input":1,"output":6},"description":"Balanced GPT-5.6 model for capable, cost-efficient everyday work","name":"GPT 5.6 Terra Pro"},"openai/gpt-chat-latest":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":30},"description":"Chat-tuned GPT model for conversational assistance, writing, and tool workflows","name":"GPT Chat Latest"},"openai/gpt-latest":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":30},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT Latest"},"openai/gpt-oss-120b":{"cost":{"input":0.35,"output":0.75},"description":"Open GPT reasoning model for self-hosted agents and controllable deployments","name":"GPT OSS 120B"},"openai/gpt-oss-20b":{"cost":{"input":0.2,"output":0.3},"description":"Open GPT reasoning model for self-hosted agents and controllable deployments","name":"GPT OSS 20B"},"openai/gpt-oss-safeguard-20b":{"cost":{"input":0.075,"output":0.3},"description":"Safety model for policy screening, moderation, and risk-aware routing workflows","name":"GPT OSS Safeguard 20B"},"openai/o1":{"cost":{"cache_read":7.5,"input":15,"output":60},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"OpenAI o1"},"openai/o1-preview":{"cost":{"cache_read":7.5,"input":15,"output":60},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"OpenAI o1-preview"},"openai/o1-pro":{"cost":{"cache_read":75,"input":150,"output":600},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"OpenAI o1 Pro"},"openai/o3":{"cost":{"cache_read":1,"input":2,"output":8},"description":"Deliberate o-series reasoner for hard math, coding, and multi-step analysis","name":"OpenAI o3"},"openai/o3-deep-research":{"cost":{"cache_read":5.5,"input":11,"output":44},"description":"Research model for long-horizon investigation, synthesis, and analytical reports","name":"OpenAI o3 Deep Research"},"openai/o3-mini":{"cost":{"cache_read":0.55,"input":1.1,"output":4.4},"description":"Smaller o-series reasoner for economical coding, math, and planning tasks","name":"OpenAI o3-mini"},"openai/o3-mini-high":{"cost":{"cache_read":0.55,"input":1.1,"output":4.4},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"OpenAI o3-mini (High)"},"openai/o3-mini-low":{"cost":{"cache_read":0.55,"input":1.1,"output":4.4},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"OpenAI o3-mini (Low)"},"openai/o3-pro-2025-06-10":{"cost":{"cache_read":11,"input":22,"output":88},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"OpenAI o3-pro (2025-06-10)"},"openai/o4-mini":{"cost":{"cache_read":0.55,"input":1.1,"output":4.4},"description":"Fast o-series model for compact reasoning, coding, and tool use","name":"OpenAI o4-mini"},"openai/o4-mini-deep-research":{"cost":{"cache_read":1.1,"input":2.2,"output":8.8},"description":"Research model for long-horizon investigation, synthesis, and analytical reports","name":"OpenAI o4-mini Deep Research"},"openai/o4-mini-high":{"cost":{"cache_read":0.55,"input":1.1,"output":4.4},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"OpenAI o4-mini high"},"pamanseau/OpenReasoning-Nemotron-32B":{"cost":{"cache_read":0.05,"input":0.1,"output":0.4},"description":"Nemotron model for efficient reasoning, coding, and specialized AI agents","name":"OpenReasoning Nemotron 32B"},"perceptron/perceptron-mk1":{"cost":{"cache_read":0.075,"input":0.15,"output":1.5},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Perceptron Mk1"},"perplexity-academic-researcher":{"cost":{"cache_read":1,"input":2,"output":8},"description":"Sonar Reasoning Pro with Perplexity's academic search mode. Prioritizes scholarly and peer-reviewed sources from academic repositories and returns cited research synthesis.","name":"Perplexity Academic Researcher"},"phi-4-mini-instruct":{"cost":{"cache_read":0.085,"input":0.17,"output":0.68},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Phi 4 Mini"},"phi-4-multimodal-instruct":{"cost":{"cache_read":0.035,"input":0.07,"output":0.11},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Phi 4 Multimodal"},"pokee-isaac":{"cost":{"cache_read":0.075,"input":0.15,"output":1},"description":"Pokee-Isaac is a 28B agentic model with a roughly 10-million-token context window, function calling, and OpenAI-compatible structured output. Pokee bills in $0.01 increments, rounding each non-zero request up to the next cent.","name":"Pokee-Isaac 28B"},"poolside/laguna-m.1":{"cost":{"cache_read":0.1,"input":0.2,"output":0.4},"description":"Poolside's open-weight model for agentic coding and long-horizon work","name":"Laguna M.1"},"poolside/laguna-s-2.1":{"cost":{"cache_read":0.01,"input":0.1,"output":0.2},"description":"Agentic coding model from Poolside in the XS size class for local deployment","name":"Laguna S 2.1"},"poolside/laguna-s-2.1:thinking":{"cost":{"cache_read":0.01,"input":0.1,"output":0.2},"description":"Agentic coding model from Poolside in the XS size class for local deployment","name":"Laguna S 2.1 Thinking"},"qvq-max":{"cost":{"cache_read":0.6,"input":1.2,"output":4.8},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen: QvQ Max"},"qwen-3.6-plus":{"cost":{"cache_read":0.0325,"cache_write":0.40625,"input":0.325,"output":1.95},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen 3.6 Plus"},"qwen-long":{"cost":{"cache_read":0.05015,"input":0.1003,"output":0.408},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen Long 10M"},"qwen-max":{"cost":{"cache_read":0.79985,"input":1.5997,"output":6.392},"description":"Flagship Qwen model for complex reasoning, coding, and agentic workflows","name":"Qwen 2.5 Max"},"qwen-plus":{"cost":{"cache_read":0.19975,"input":0.3995,"output":1.2002},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen Plus"},"qwen-turbo":{"cost":{"cache_read":0.02499,"input":0.04998,"output":0.2006},"description":"Efficient Qwen model for fast chat, extraction, and high-volume workloads","name":"Qwen Turbo"},"qwen/Qwen2.5-Coder-32B-Instruct":{"cost":{"cache_read":0.1003,"input":0.2006,"output":0.2006},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen 2.5 Coder 32b"},"qwen/Qwen3-235B-A22B-Instruct-2507":{"cost":{"cache_read":0.065,"input":0.13,"output":0.5},"description":"Updated large open Qwen3 MoE instruct model for multilingual chat, coding, and tool use","name":"Qwen 3 235b A22B 2507"},"qwen/Qwen3-235B-A22B-Instruct-2507-TEE":{"cost":{"cache_read":0.065,"input":0.13,"output":0.5},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen 3 235b A22B 2507 (TEE)"},"qwen/Qwen3-235B-A22B-Thinking-2507":{"cost":{"cache_read":0.15,"input":0.3,"output":0.5},"description":"Qwen reasoning model for deliberate problem solving, math, and coding","name":"Qwen 3 235b A22B 2507 Thinking"},"qwen/Qwen3-8B":{"cost":{"cache_read":0.235,"input":0.47,"output":0.47},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen 3 8B"},"qwen/Qwen3-Next-80B-A3B-Instruct":{"cost":{"cache_read":0.075,"input":0.15,"output":0.65},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3 Next 80B A3B (Instruct)"},"qwen/Qwen3-VL-235B-A22B-Instruct":{"cost":{"cache_read":0.15,"input":0.3,"output":1.2},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3 VL 235B A22B Instruct"},"qwen/Qwen3.6-35B-A3B":{"cost":{"cache_read":0.056,"input":0.112,"output":0.8},"description":"Open multimodal Qwen MoE for local agents that need vision, audio, and code","name":"Qwen3.6 35B A3B"},"qwen/Qwen3.6-35B-A3B:thinking":{"cost":{"cache_read":0.056,"input":0.112,"output":0.8},"description":"Open multimodal Qwen MoE for local agents that need vision, audio, and code","name":"Qwen3.6 35B A3B Thinking"},"qwen/qwen-2.5-72b-instruct":{"cost":{"cache_read":0.1785,"input":0.357,"output":0.408},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen2.5 72B"},"qwen/qwen3-14b":{"cost":{"cache_read":0.04,"input":0.08,"output":0.24},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen 3 14b"},"qwen/qwen3-235b-a22b":{"cost":{"cache_read":0.15,"input":0.3,"output":0.5},"description":"Large open Qwen MoE for multilingual reasoning, coding, and tool use","name":"Qwen 3 235b A22B"},"qwen/qwen3-30b-a3b":{"cost":{"cache_read":0.05,"input":0.1,"output":0.3},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3 30B A3B"},"qwen/qwen3-32b":{"cost":{"cache_read":0.05,"input":0.1,"output":0.3},"description":"Dense open Qwen model for self-hosted chat, reasoning, and coding","name":"Qwen 3 32b"},"qwen/qwen3-coder":{"cost":{"cache_read":0.065,"input":0.13,"output":0.5},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen 3 Coder 480B"},"qwen/qwen3-coder-flash":{"cost":{"cache_read":0.15,"input":0.3,"output":1.5},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen3 Coder Flash"},"qwen/qwen3-coder-next":{"cost":{"cache_read":0.1,"input":0.2,"output":1.5},"description":"Open-weight Qwen coding model for agents, repository edits, and multi-turn tool use","name":"Qwen3 Coder Next"},"qwen/qwen3-coder-plus":{"cost":{"cache_read":0.5,"input":1,"output":5},"description":"Hosted Qwen coder for software agents, repo edits, and long-context code","name":"Qwen3 Coder Plus"},"qwen/qwen3-max":{"cost":{"cache_read":0.6001,"input":1.2002,"output":6.001},"description":"Flagship Qwen3 model for coding agents, complex reasoning, and tool use","name":"Qwen3 Max"},"qwen/qwen3-next-80b-a3b-thinking":{"cost":{"cache_read":0.075,"input":0.15,"output":0.65},"description":"Efficient Qwen thinking model for local reasoning, math, and coding agents","name":"Qwen3 Next 80B A3B (Thinking)"},"qwen/qwen3.5-397b-a17b":{"cost":{"cache_read":0.3,"input":0.6,"output":3.6},"description":"Large open Qwen multimodal MoE for visual agents and long technical tasks","name":"Qwen3.5 397B A17B"},"qwen/qwen3.5-397b-a17b-thinking":{"cost":{"cache_read":0.3,"input":0.6,"output":3.6},"description":"Large open Qwen multimodal MoE for visual agents and long technical tasks","name":"Qwen3.5 397B A17B Thinking"},"qwen/qwen3.5-9b":{"cost":{"cache_read":0.025,"input":0.05,"output":0.15},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3.5 9B"},"qwen/qwen3.5-plus":{"cost":{"cache_read":0.04,"input":0.4,"output":2.4},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 Plus"},"qwen/qwen3.5-plus-thinking":{"cost":{"cache_read":0.04,"input":0.4,"output":2.4},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 Plus Thinking"},"qwen25-vl-72b-instruct":{"cost":{"cache_read":0.349945,"input":0.69989,"output":0.69989},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen25 VL 72b"},"qwen3-30b-a3b-instruct-2507":{"cost":{"cache_read":0.1,"input":0.2,"output":0.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3 30B A3B Instruct 2507"},"qwen3-coder-30b-a3b-instruct":{"cost":{"cache_read":0.05,"input":0.1,"output":0.4},"description":"Smaller Qwen coder for efficient local agents and repo-level fixes","name":"Qwen3 Coder 30B A3B Instruct"},"qwen3-max-2026-01-23":{"cost":{"cache_read":0.6001,"input":1.2002,"output":6.001},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3 Max 2026-01-23"},"qwen3-vl-235b-a22b-instruct-original":{"cost":{"cache_read":0.25,"input":0.5,"output":1.2},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3 VL 235B A22B Instruct Original"},"qwen3-vl-235b-a22b-thinking":{"cost":{"cache_read":0.25,"input":0.5,"output":6},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Qwen3 VL 235B A22B Thinking"},"qwen3.5-122b-a10b":{"cost":{"cache_read":0.103788,"input":0.437,"output":3.496},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 122B A10B"},"qwen3.5-122b-a10b:thinking":{"cost":{"cache_read":0.103788,"input":0.437,"output":3.496},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 122B A10B Thinking"},"qwen3.5-27b":{"cost":{"cache_read":0.135,"input":0.27,"output":2.16},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 27B"},"qwen3.5-27b:thinking":{"cost":{"cache_read":0.135,"input":0.27,"output":2.16},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 27B Thinking"},"qwen3.5-35b-a3b":{"cost":{"cache_read":0.1125,"input":0.225,"output":1.8},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 35B A3B"},"qwen3.5-35b-a3b:thinking":{"cost":{"cache_read":0.1125,"input":0.225,"output":1.8},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 35B A3B Thinking"},"qwen3.5-flash":{"cost":{"cache_read":0.05,"input":0.1,"output":0.4},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 Flash"},"qwen3.5-flash:thinking":{"cost":{"cache_read":0.05,"input":0.1,"output":0.4},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 Flash Thinking"},"qwen3.5-omni-flash":{"cost":{"input":0,"output":0},"description":"Omni-modal model for text, vision, audio, and multimodal agent tasks","name":"Qwen3.5 Omni Flash"},"qwen3.5-omni-plus":{"cost":{"input":0,"output":0},"description":"Omni-modal model for text, vision, audio, and multimodal agent tasks","name":"Qwen3.5 Omni Plus"},"qwen3.6-max-preview":{"cost":{"cache_read":0.52,"input":1.04,"output":6.24},"description":"Flagship Qwen model for complex reasoning, coding, and agentic workflows","name":"Qwen3.6 Max Preview"},"qwen3.7-flash":{"cost":{"cache_read":0.006,"cache_write":0.038,"input":0.03,"output":0.13},"description":"Lightweight multimodal Qwen model for high-throughput text, image, and video tasks","name":"Qwen3.7 Flash"},"qwen3.7-flash:thinking":{"cost":{"cache_read":0.006,"cache_write":0.038,"input":0.03,"output":0.13},"description":"Lightweight multimodal Qwen model for high-throughput text, image, and video tasks","name":"Qwen3.7 Flash Thinking"},"qwen3.7-max":{"cost":{"cache_read":0.5,"cache_write":3.125,"input":2.5,"output":7.5},"description":"Qwen frontier model tuned for agent frameworks, coding assistants, and long tasks","name":"Qwen3.7 Max"},"qwen3.7-max:thinking":{"cost":{"cache_read":0.5,"cache_write":3.125,"input":2.5,"output":7.5},"description":"Qwen frontier model tuned for agent frameworks, coding assistants, and long tasks","name":"Qwen3.7 Max Thinking"},"qwen3.7-plus":{"cost":{"cache_read":0.08,"cache_write":0.5,"input":0.4,"output":1.6},"description":"Multimodal Qwen workhorse for long-context agents, visual inputs, and coding","name":"Qwen3.7 Plus"},"qwen3.7-plus:thinking":{"cost":{"cache_read":0.08,"cache_write":0.5,"input":0.4,"output":1.6},"description":"Multimodal Qwen workhorse for long-context agents, visual inputs, and coding","name":"Qwen3.7 Plus Thinking"},"qwen3.8-max":{"cost":{"cache_read":0.25,"cache_write":2.5,"input":2,"output":6},"description":"2.4-trillion-parameter MoE flagship for coding, professional work, multimodal understanding, and long-horizon agentic workflows","name":"Qwen3.8 Max"},"qwen3.8-max-preview":{"cost":{"cache_read":0.15,"cache_write":2,"input":1.5,"output":5},"description":"Preview Qwen flagship for million-token multimodal reasoning and long-horizon agentic workflows","name":"Qwen3.8 Max Preview"},"qwen3.8-max:thinking":{"cost":{"cache_read":0.25,"cache_write":2.5,"input":2,"output":6},"description":"2.4-trillion-parameter MoE flagship for coding, professional work, multimodal understanding, and long-horizon agentic workflows","name":"Qwen3.8 Max Thinking"},"sakana/fugu-ultra":{"cost":{"cache_read":0.525,"input":5.25,"output":31.5},"description":"Quality-first multi-agent model for hard research, analysis, and competitions","name":"Fugu Ultra"},"sakana/fugu-ultra-v1.1":{"cost":{"cache_read":0.525,"input":5.25,"output":31.5},"description":"Sakana AI's upgraded Fugu Ultra release with stronger coding, agentic task execution, and advanced reasoning through dynamic orchestration of frontier models.","name":"Fugu Ultra v1.1"},"sarvam-105b":{"cost":{"cache_read":0.028,"input":0.045,"output":0.177},"description":"Flagship Indian-language reasoning model for enterprise multilingual applications","name":"Sarvam 105B"},"sarvam-30b":{"cost":{"cache_read":0.017,"input":0.028,"output":0.111},"description":"Efficient Indian-language reasoning model for chat, coding, and multilingual work","name":"Sarvam 30B"},"shisa-ai/shisa-v2-llama3.3-70b":{"cost":{"cache_read":0.25,"input":0.5,"output":0.5},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Shisa V2 Llama 3.3 70B"},"shisa-ai/shisa-v2.1-llama3.3-70b":{"cost":{"cache_read":0.25,"input":0.5,"output":0.5},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Shisa V2.1 Llama 3.3 70B"},"sonar":{"cost":{"cache_read":0.5,"input":1,"output":1},"description":"Fast web-grounded Sonar for current answers, citations, and lightweight retrieval","name":"Perplexity Simple"},"sonar-deep-research":{"cost":{"cache_read":1.7,"input":3.4,"output":13.6},"description":"Research model for long-horizon investigation, synthesis, and analytical reports","name":"Perplexity Deep Research"},"sonar-pro":{"cost":{"cache_read":1.5,"input":3,"output":15},"description":"Deeper Sonar search model with broader retrieval and stronger synthesis","name":"Perplexity Pro"},"sonar-reasoning-pro":{"cost":{"cache_read":1,"input":2,"output":8},"description":"Web-grounded Sonar for multi-step research questions that need cited reasoning","name":"Perplexity Reasoning Pro"},"soob3123/GrayLine-Qwen3-8B":{"cost":{"cache_read":0.15,"input":0.3,"output":0.3},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Grayline Qwen3 8B"},"soob3123/Veiled-Calla-12B":{"cost":{"cache_read":0.15,"input":0.3,"output":0.3},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Veiled Calla 12B"},"soob3123/amoral-gemma3-27B-v2":{"cost":{"cache_read":0.15,"input":0.3,"output":0.3},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Amoral Gemma3 27B v2"},"step-2-16k-exp":{"cost":{"cache_read":3.502,"input":7.004,"output":19.992},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Step-2 16k Exp"},"step-2-mini":{"cost":{"cache_read":0.1003,"input":0.2006,"output":0.408},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Step-2 Mini"},"step-3":{"cost":{"cache_read":0.12495,"input":0.2499,"output":0.6494},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Step-3"},"step-r1-v-mini":{"cost":{"cache_read":1.25,"input":2.5,"output":11},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Step R1 V Mini"},"stepfun-ai/step-3.5-flash":{"cost":{"cache_read":0.05,"input":0.1,"output":0.3},"description":"StepFun flash lane for quick multimodal reasoning and coding assistance","name":"Step 3.5 Flash"},"stepfun-ai/step-3.5-flash-2603":{"cost":{"cache_read":0.05,"input":0.1,"output":0.3},"description":"StepFun flash model for efficient multimodal reasoning, coding, and tool use","name":"Step 3.5 Flash 2603"},"stepfun/step-3.7-flash:thinking":{"cost":{"cache_read":0.04,"input":0.2,"output":1.15},"description":"Newer StepFun flash model for faster agents, coding, and multimodal prompts","name":"Step 3.7 Flash Thinking"},"tencent/Hunyuan-MT-7B":{"cost":{"cache_read":5,"input":10,"output":20},"description":"Translation model for multilingual conversion, localization, and cross-language workflows","name":"Hunyuan MT 7B"},"tencent/hy3":{"cost":{"cache_read":0.029,"input":0.066,"output":0.26},"description":"Tencent Hy reasoning model for coding, instruction following, and agent tasks","name":"Tencent Hy3"},"thinkingmachines/Inkling-Small":{"cost":{"cache_read":0.1,"input":0.5,"output":1.2},"description":"Multimodal MoE reasoning model (276B total, 12B active) for text, image, and audio","name":"Inkling Small"},"thinkingmachines/Inkling-Small:thinking":{"cost":{"cache_read":0.1,"input":0.5,"output":1.2},"description":"Multimodal MoE reasoning model (276B total, 12B active) for text, image, and audio","name":"Inkling Small Thinking"},"thinkingmachines/inkling":{"cost":{"cache_read":0.17,"input":1,"output":4.05},"description":"Multimodal MoE reasoning model (975B total, 41B active) for text, image, and audio","name":"Inkling"},"thinkingmachines/inkling:thinking":{"cost":{"cache_read":0.17,"input":1,"output":4.05},"description":"Multimodal MoE reasoning model (975B total, 41B active) for text, image, and audio","name":"Inkling Thinking"},"undi95/remm-slerp-l2-13b":{"cost":{"cache_read":0.3995,"input":0.799,"output":1.207},"description":"Open Llama multimodal model for image understanding and text reasoning","name":"ReMM SLERP 13B"},"universal-summarizer":{"cost":{"input":30,"output":30},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Universal Summarizer"},"unsloth/gemma-3-12b-it":{"cost":{"cache_read":0.136,"input":0.272,"output":0.272},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 3 12B IT"},"unsloth/gemma-3-27b-it":{"cost":{"cache_read":0.1496,"input":0.2992,"output":0.2992},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 3 27B IT"},"unsloth/gemma-3-4b-it":{"cost":{"cache_read":0.1003,"input":0.2006,"output":0.2006},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 3 4B IT"},"upstage/solar-pro-3":{"cost":{"cache_read":0.015,"input":0.15,"output":0.6},"description":"Flagship model for demanding analysis, coding, and production agent workflows","name":"Solar Pro 3"},"venice-uncensored":{"cost":{"cache_read":0.2,"input":0.4,"output":0.4},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Venice Uncensored"},"x-ai/grok-4.20":{"cost":{"cache_read":1,"input":2,"output":6},"description":"Grok model for agentic tool use, reasoning, coding, and live assistance","name":"Grok 4.20"},"x-ai/grok-4.20-multi-agent":{"cost":{"cache_read":1,"input":2,"output":6},"description":"Grok model for agentic tool use, reasoning, coding, and live assistance","name":"Grok 4.20 Multi-Agent"},"x-ai/grok-4.3":{"cost":{"cache_read":0.2,"input":1.25,"output":2.5},"description":"xAI's default Grok for chat, coding, agentic tools, and lower hallucination risk","name":"Grok 4.3"},"x-ai/grok-4.5":{"cost":{"cache_read":0.5,"input":2,"output":6},"description":"xAI's latest Grok for chat, coding, agentic tools, and lower hallucination risk","name":"Grok 4.5"},"x-ai/grok-build-0.1":{"cost":{"cache_read":0.2,"input":1,"output":2},"description":"Fast Grok coding model tuned for agentic engineering and iterative edits","name":"Grok Build 0.1"},"x-ai/grok-latest":{"cost":{"cache_read":0.5,"input":2,"output":6},"description":"Grok model for agentic tool use, reasoning, coding, and live assistance","name":"Grok Latest"},"xiaomi/mimo-v2.5":{"cost":{"cache_read":0.0028,"cache_write":0,"input":0.14,"output":0.28},"description":"Open MiMo model for multimodal coding agents and long-context automation","name":"MiMo V2.5"},"xiaomi/mimo-v2.5-pro":{"cost":{"cache_read":0.0036,"cache_write":0,"input":0.435,"output":0.87},"description":"Stronger MiMo Pro tier for multimodal reasoning and coding-agent execution","name":"MiMo V2.5 Pro"},"xiaomi/mimo-v2.5-pro-crof":{"cost":{"cache_read":0.003,"input":0.4,"output":0.8},"description":"MiMo V2.5 Pro is Xiaomi's long-context flagship general model for coding and agentic orchestration. This separately served variant is intended for users concerned about censorship on the regular Xiaomi MiMo V2.5 Pro, and it is included in the NanoGPT subscription.","name":"MiMo V2.5 Pro (Crof)"},"xiaomi/mimo-v2.5-pro-crof:thinking":{"cost":{"cache_read":0.003,"input":0.4,"output":0.8},"description":"MiMo V2.5 Pro with Xiaomi thinking enabled for coding, long-context reasoning, and agentic orchestration. This separately served thinking variant is intended for users concerned about censorship on the regular Xiaomi MiMo V2.5 Pro, and it is included in the NanoGPT subscription.","name":"MiMo V2.5 Pro Thinking (Crof)"},"xiaomi/mimo-v2.5-pro:thinking":{"cost":{"cache_read":0.0036,"cache_write":0,"input":0.435,"output":0.87},"description":"Stronger MiMo Pro tier for multimodal reasoning and coding-agent execution","name":"MiMo V2.5 Pro Thinking"},"xiaomi/mimo-v2.5:thinking":{"cost":{"cache_read":0.0028,"cache_write":0,"input":0.14,"output":0.28},"description":"Open MiMo model for multimodal coding agents and long-context automation","name":"MiMo V2.5 Thinking"},"yi-large":{"cost":{"cache_read":1.598,"input":3.196,"output":3.196},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Yi Large"},"yi-lightning":{"cost":{"cache_read":0.1003,"input":0.2006,"output":0.2006},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Yi Lightning"},"yi-medium-200k":{"cost":{"cache_read":1.2495,"input":2.499,"output":2.499},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"Yi Medium 200k"},"z-ai/glm-4.5v":{"cost":{"cache_read":0.3,"input":0.6,"output":1.8},"description":"GLM vision model for visual reasoning, documents, and multimodal agents","name":"GLM 4.5V"},"z-ai/glm-4.5v:thinking":{"cost":{"cache_read":0.3,"input":0.6,"output":1.8},"description":"GLM vision model for visual reasoning, documents, and multimodal agents","name":"GLM 4.5V Thinking"},"z-ai/glm-4.6":{"cost":{"cache_read":0.175,"input":0.35,"output":1.4},"description":"Late GLM-4 workhorse for coding agents, reasoning, and structured tasks","name":"GLM 4.6"},"z-ai/glm-4.6:thinking":{"cost":{"cache_read":0.175,"input":0.35,"output":1.4},"description":"Late GLM-4 workhorse for coding agents, reasoning, and structured tasks","name":"GLM 4.6 Thinking"},"z-ai/glm-5-turbo":{"cost":{"cache_read":0.24,"input":1.2,"output":4},"description":"Faster GLM-5 lane for coding agents that need lower latency","name":"GLM 5 Turbo"},"z-ai/glm-5v-turbo":{"cost":{"cache_read":0.24,"input":1.2,"output":4},"description":"Fast GLM vision model for screenshots, documents, and multimodal agent tasks","name":"GLM 5V Turbo"},"z-ai/glm-5v-turbo:thinking":{"cost":{"cache_read":0.24,"input":1.2,"output":4},"description":"Fast GLM vision model for screenshots, documents, and multimodal agent tasks","name":"GLM 5V Turbo Thinking"},"zai-org/GLM-4.5-Air":{"cost":{"cache_read":0.06,"input":0.12,"output":0.8},"description":"Lighter GLM-4.5 variant for fast coding assistance and cheaper agents","name":"GLM 4.5 Air"},"zai-org/GLM-4.5-Air:thinking":{"cost":{"cache_read":0.06,"input":0.12,"output":0.8},"description":"Lighter GLM-4.5 variant for fast coding assistance and cheaper agents","name":"GLM 4.5 Air (Thinking)"},"zai-org/GLM-4.5:thinking":{"cost":{"cache_read":0.15,"input":0.3,"output":1.3},"description":"Hybrid-reasoning GLM release that made the 4.5 line broadly useful","name":"GLM 4.5 (Thinking)"},"zai-org/GLM-4.6-turbo":{"cost":{"cache_read":0.5,"input":1,"output":3},"description":"Efficient GLM model for fast reasoning, coding, and agent workflows","name":"GLM 4.6 Turbo"},"zai-org/GLM-4.6-turbo:thinking":{"cost":{"cache_read":0.5,"input":1,"output":3},"description":"Efficient GLM model for fast reasoning, coding, and agent workflows","name":"GLM 4.6 Turbo (Thinking)"},"zai-org/glm-4.5":{"cost":{"cache_read":0.15,"input":0.3,"output":1.3},"description":"Hybrid-reasoning GLM release that made the 4.5 line broadly useful","name":"GLM 4.5"},"zai-org/glm-4.6-original":{"cost":{"cache_read":0.175,"input":0.35,"output":1.4},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM 4.6 Original"},"zai-org/glm-4.6v":{"cost":{"cache_read":0.15,"input":0.3,"output":0.9},"description":"GLM vision model for visual reasoning, documents, and multimodal agents","name":"GLM 4.6V"},"zai-org/glm-4.6v-flash-original":{"cost":{"cache_read":0.05,"input":0.1,"output":0.4},"description":"GLM vision model for visual reasoning, documents, and multimodal agents","name":"GLM 4.6V Flash"},"zai-org/glm-4.6v-original":{"cost":{"cache_read":0.3,"input":0.6,"output":0.9},"description":"GLM vision model for visual reasoning, documents, and multimodal agents","name":"GLM 4.6V Original"},"zai-org/glm-4.7":{"cost":{"cache_read":0.1,"input":0.2,"output":0.8},"description":"Mature GLM model for dependable coding, reasoning, and structured agent tasks","name":"GLM 4.7"},"zai-org/glm-4.7-flash":{"cost":{"cache_read":0.035,"input":0.07,"output":0.4},"description":"Budget GLM lane for fast coding help, routing, and everyday automation","name":"GLM 4.7 Flash"},"zai-org/glm-4.7-flash-original":{"cost":{"cache_read":0.035,"input":0.07,"output":0.4},"description":"Efficient GLM model for fast reasoning, coding, and agent workflows","name":"GLM 4.7 Flash Original"},"zai-org/glm-4.7-flash-original:thinking":{"cost":{"cache_read":0.035,"input":0.07,"output":0.4},"description":"Efficient GLM model for fast reasoning, coding, and agent workflows","name":"GLM 4.7 Flash Original Thinking"},"zai-org/glm-4.7-flash:thinking":{"cost":{"cache_read":0.035,"input":0.07,"output":0.4},"description":"Budget GLM lane for fast coding help, routing, and everyday automation","name":"GLM 4.7 Flash Thinking"},"zai-org/glm-4.7-original":{"cost":{"cache_read":0.11,"input":0.6,"output":2.2},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM 4.7 Original"},"zai-org/glm-4.7-original:thinking":{"cost":{"cache_read":0.11,"input":0.6,"output":2.2},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM 4.7 Original Thinking"},"zai-org/glm-4.7:thinking":{"cost":{"cache_read":0.1,"input":0.2,"output":0.8},"description":"Mature GLM model for dependable coding, reasoning, and structured agent tasks","name":"GLM 4.7 Thinking"},"zai-org/glm-5":{"cost":{"cache_read":0.13,"input":0.5,"output":2.55},"description":"General GLM flagship for coding, analysis, and tool-heavy engineering workflows","name":"GLM 5"},"zai-org/glm-5-original":{"cost":{"cache_read":0.2,"input":1,"output":3.2},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM 5 Original"},"zai-org/glm-5-original:thinking":{"cost":{"cache_read":0.2,"input":1,"output":3.2},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM 5 Original Thinking"},"zai-org/glm-5.1":{"cost":{"cache_read":0.15,"input":0.75,"output":2.6},"description":"Strong GLM coding model for agentic engineering, terminals, and repository generation","name":"GLM 5.1"},"zai-org/glm-5.1:thinking":{"cost":{"cache_read":0.15,"input":0.75,"output":2.6},"description":"Strong GLM coding model for agentic engineering, terminals, and repository generation","name":"GLM 5.1 Thinking"},"zai-org/glm-5.2":{"cost":{"cache_read":0.078,"input":0.42,"output":1.32},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM 5.2"},"zai-org/glm-5.2:thinking":{"cost":{"cache_read":0.078,"input":0.42,"output":1.32},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM 5.2 Thinking"},"zai-org/glm-5:thinking":{"cost":{"cache_read":0.13,"input":0.5,"output":2.55},"description":"General GLM flagship for coding, analysis, and tool-heavy engineering workflows","name":"GLM 5 Thinking"},"zai-org/glm-latest":{"cost":{"cache_read":0.078,"input":0.42,"output":1.32},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM Latest"}}},"nebius":{"models":{"MiniMaxAI/MiniMax-M2.5":{"cost":{"cache_read":0.03,"cache_write":0.375,"input":0.3,"output":1.2},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"MiniMax-M2.5"},"MiniMaxAI/MiniMax-M2.5-fast":{"cost":{"cache_read":0.03,"cache_write":0.375,"input":0.3,"output":1.2},"description":"Legacy model retained for compatibility with older integrations","name":"MiniMax-M2.5-fast"},"MiniMaxAI/MiniMax-M3":{"cost":{"input":0.3,"output":1.2},"description":"MiniMax multimodal model for long-context coding, perception, and agent planning","name":"MiniMax-M3"},"NousResearch/Hermes-4-405B":{"cost":{"cache_read":0.1,"cache_write":1.25,"input":1.0,"output":3.0,"reasoning":3.0},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"Hermes-4-405B"},"NousResearch/Hermes-4-70B":{"cost":{"cache_read":0.013,"cache_write":0.16,"input":0.13,"output":0.4,"reasoning":0.4},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"Hermes-4-70B"},"PrimeIntellect/INTELLECT-3":{"cost":{"cache_read":0.02,"cache_write":0.25,"input":0.2,"output":1.1},"description":"Legacy model retained for compatibility with older integrations","name":"INTELLECT-3"},"Qwen/Qwen2.5-VL-72B-Instruct":{"cost":{"cache_read":0.025,"cache_write":0.31,"input":0.25,"output":0.75},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen2.5-VL-72B-Instruct"},"Qwen/Qwen3-235B-A22B-Instruct-2507":{"cost":{"input":0.2,"output":0.6},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3 235B A22B Instruct 2507"},"Qwen/Qwen3-235B-A22B-Thinking-2507-fast":{"cost":{"cache_read":0.05,"cache_write":0.625,"input":0.5,"output":2.0},"description":"Legacy model retained for compatibility with older integrations","name":"Qwen3-235B-A22B-Thinking-2507-fast"},"Qwen/Qwen3-30B-A3B-Instruct-2507":{"cost":{"cache_read":0.01,"cache_write":0.125,"input":0.1,"output":0.3},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3-30B-A3B-Instruct-2507"},"Qwen/Qwen3-32B":{"cost":{"cache_read":0.01,"cache_write":0.125,"input":0.1,"output":0.3},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3-32B"},"Qwen/Qwen3-Embedding-8B":{"cost":{"input":0.01,"output":0.0},"description":"Embedding model for semantic search, retrieval, clustering, and ranking pipelines","name":"Qwen3-Embedding-8B"},"Qwen/Qwen3-Next-80B-A3B-Thinking":{"cost":{"cache_read":0.015,"cache_write":0.18,"input":0.15,"output":1.2,"reasoning":1.2},"description":"Qwen reasoning model for deliberate problem solving, math, and coding","name":"Qwen3-Next-80B-A3B-Thinking"},"Qwen/Qwen3-Next-80B-A3B-Thinking-fast":{"cost":{"cache_read":0.015,"cache_write":0.1875,"input":0.15,"output":1.2},"description":"Legacy model retained for compatibility with older integrations","name":"Qwen3-Next-80B-A3B-Thinking-fast"},"Qwen/Qwen3.5-397B-A17B":{"cost":{"cache_read":0.06,"cache_write":0.75,"input":0.6,"output":3.6},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3.5-397B-A17B"},"Qwen/Qwen3.5-397B-A17B-fast":{"cost":{"cache_read":0.06,"cache_write":0.75,"input":0.6,"output":3.6},"description":"Legacy model retained for compatibility with older integrations","name":"Qwen3.5-397B-A17B-fast"},"deepseek-ai/DeepSeek-V3.2":{"cost":{"cache_read":0.03,"cache_write":0.375,"input":0.3,"output":0.45,"reasoning":0.45},"description":"Legacy model retained for compatibility with older integrations","name":"DeepSeek-V3.2"},"deepseek-ai/DeepSeek-V3.2-fast":{"cost":{"cache_read":0.04,"cache_write":0.5,"input":0.4,"output":2.0},"description":"Legacy model retained for compatibility with older integrations","name":"DeepSeek-V3.2-fast"},"deepseek-ai/DeepSeek-V4-Pro":{"cost":{"cache_read":0.15,"input":1.75,"output":3.5},"description":"Open MoE flagship with million-token context for coding and long agent runs","name":"DeepSeek V4 Pro"},"google/gemma-3-27b-it":{"cost":{"cache_read":0.01,"cache_write":0.125,"input":0.1,"output":0.3},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma-3-27b-it"},"meta-llama/Llama-3.3-70B-Instruct":{"cost":{"cache_read":0.013,"cache_write":0.16,"input":0.13,"output":0.4},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama-3.3-70B-Instruct"},"moonshotai/Kimi-K2.5":{"cost":{"cache_read":0.05,"cache_write":0.625,"input":0.5,"output":2.5,"reasoning":2.5},"description":"Legacy model retained for compatibility with older integrations","name":"Kimi-K2.5"},"moonshotai/Kimi-K2.5-fast":{"cost":{"cache_read":0.05,"cache_write":0.625,"input":0.5,"output":2.5},"description":"Legacy model retained for compatibility with older integrations","name":"Kimi-K2.5-fast"},"moonshotai/Kimi-K2.7-Code":{"cost":{"input":0.95,"output":4},"description":"Coding-focused Kimi model, stronger on long-horizon repo work with less overthinking","name":"Kimi K2.7 Code"},"moonshotai/Kimi-K3":{"cost":{"cache_read":3,"input":3,"output":15},"description":"Multimodal Kimi model with 1M context and toggleable max-effort thinking for long-horizon agent work","name":"Kimi K3"},"nvidia/Llama-3_1-Nemotron-Ultra-253B-v1":{"cost":{"cache_read":0.06,"cache_write":0.75,"input":0.6,"output":1.8},"description":"Flagship Nemotron model for high-throughput reasoning and complex agents","name":"Llama-3.1-Nemotron-Ultra-253B-v1"},"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B":{"cost":{"cache_read":0.006,"cache_write":0.075,"input":0.06,"output":0.24},"description":"Small Nemotron 3 MoE for efficient coding, math, and long-context agents","name":"Nemotron-3-Nano-30B-A3B"},"nvidia/Nemotron-3-Nano-Omni":{"cost":{"cache_read":0.006,"cache_write":0.075,"input":0.06,"output":0.24},"description":"Open Nemotron omni model combining reasoning with text, vision, and audio","name":"Nemotron-3-Nano-Omni"},"nvidia/nemotron-3-super-120b-a12b":{"cost":{"input":0.3,"output":0.9},"description":"Nemotron middle tier for collaborative agents and high-volume reasoning workloads","name":"Nemotron-3-Super-120B-A12B"},"openai/gpt-oss-120b":{"cost":{"cache_read":0.015,"cache_write":0.18,"input":0.15,"output":0.6,"reasoning":0.6},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"gpt-oss-120b"},"openai/gpt-oss-120b-fast":{"cost":{"cache_read":0.01,"cache_write":0.125,"input":0.1,"output":0.5},"description":"Legacy model retained for compatibility with older integrations","name":"gpt-oss-120b-fast"},"zai-org/GLM-5":{"cost":{"cache_read":0.1,"cache_write":1.0,"input":1.0,"output":3.2},"description":"Legacy model retained for compatibility with older integrations","name":"GLM-5"},"zai-org/GLM-5.2":{"cost":{"input":1.4,"output":4.4},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM-5.2"}}},"openai":{"models":{"chatgpt-image-latest":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"chatgpt-image-latest"},"gpt-3.5-turbo":{"cost":{"cache_read":0,"input":0.5,"output":1.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-3.5-turbo"},"gpt-4":{"cost":{"input":30.0,"output":60.0},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-4"},"gpt-4-turbo":{"cost":{"input":10.0,"output":30.0},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-4 Turbo"},"gpt-4.1":{"cost":{"cache_read":0.5,"input":2.0,"output":8.0},"description":"Long-lived GPT workhorse for coding, instruction following, and production apps","name":"GPT-4.1"},"gpt-4.1-mini":{"cost":{"cache_read":0.1,"input":0.4,"output":1.6},"description":"Affordable GPT-4.1 lane for fast coding help and structured extraction","name":"GPT-4.1 mini"},"gpt-4.1-nano":{"cost":{"cache_read":0.025,"input":0.1,"output":0.4},"description":"Tiny GPT-4.1 option for classification, routing, and very high-volume tasks","name":"GPT-4.1 nano"},"gpt-4o":{"cost":{"cache_read":1.25,"input":2.5,"output":10.0},"description":"Omni-era GPT for multimodal chat, practical coding, and general assistants","name":"GPT-4o"},"gpt-4o-2024-05-13":{"cost":{"input":5.0,"output":15.0},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-4o (2024-05-13)"},"gpt-4o-2024-08-06":{"cost":{"cache_read":1.25,"input":2.5,"output":10.0},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-4o (2024-08-06)"},"gpt-4o-2024-11-20":{"cost":{"cache_read":1.25,"input":2.5,"output":10.0},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-4o (2024-11-20)"},"gpt-4o-mini":{"cost":{"cache_read":0.075,"input":0.15,"output":0.6},"description":"Small omni GPT for cheap multimodal assistance and production-scale traffic","name":"GPT-4o mini"},"gpt-5":{"cost":{"cache_read":0.125,"input":1.25,"output":10.0},"description":"Original GPT-5 workhorse for reasoning, coding, writing, and tool workflows","name":"GPT-5"},"gpt-5-mini":{"cost":{"cache_read":0.025,"input":0.25,"output":2.0},"description":"Small GPT-5 for responsive agents, coding help, and everyday automation","name":"GPT-5 Mini"},"gpt-5-nano":{"cost":{"cache_read":0.005,"input":0.05,"output":0.4},"description":"Tiny GPT-5 lane for routing, extraction, classification, and bulk jobs","name":"GPT-5 Nano"},"gpt-5-pro":{"cost":{"input":15.0,"output":120.0},"description":"Higher-accuracy GPT-5 tier for tough analysis, coding reviews, and planning","name":"GPT-5 Pro"},"gpt-5.1":{"cost":{"cache_read":0.125,"input":1.25,"output":10.0},"description":"Sharper GPT-5 generation for coding, product work, and tool-assisted tasks","name":"GPT-5.1"},"gpt-5.2":{"cost":{"cache_read":0.175,"input":1.75,"output":14.0},"description":"Reliable GPT generation for broad coding, writing, and tool-assisted product work","name":"GPT-5.2"},"gpt-5.2-chat-latest":{"cost":{"cache_read":0.175,"input":1.75,"output":14.0},"description":"Chat-tuned GPT model for conversational assistance, writing, and tool workflows","name":"GPT-5.2 Chat"},"gpt-5.2-pro":{"cost":{"input":21.0,"output":168.0},"description":"Higher-accuracy GPT-5.2 variant for tougher reasoning and review workflows","name":"GPT-5.2 Pro"},"gpt-5.3-chat-latest":{"cost":{"cache_read":0.175,"input":1.75,"output":14.0},"description":"Chat-tuned GPT model for conversational assistance, writing, and tool workflows","name":"GPT-5.3 Chat (latest)"},"gpt-5.3-codex":{"cost":{"cache_read":0.175,"input":1.75,"output":14.0},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.3 Codex"},"gpt-5.3-codex-spark":{"cost":{"cache_read":0.175,"input":1.75,"output":14.0},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.3 Codex Spark"},"gpt-5.4":{"cost":{"cache_read":0.25,"input":2.5,"output":15.0,"tiers":[{"cache_read":0.5,"input":5.0,"output":22.5,"tier":{"size":272000}}]},"description":"Agent-ready GPT for coding and computer-use workflows at a lower cost","name":"GPT-5.4"},"gpt-5.4-mini":{"cost":{"cache_read":0.075,"input":0.75,"output":4.5},"description":"Strong small GPT for coding subagents, quick tool use, and high-volume work","name":"GPT-5.4 mini"},"gpt-5.4-nano":{"cost":{"cache_read":0.02,"input":0.2,"output":1.25},"description":"Cheapest GPT-5.4 lane for simple routing, extraction, and bulk automation","name":"GPT-5.4 nano"},"gpt-5.4-pro":{"cost":{"input":30.0,"output":180.0,"tiers":[{"input":60.0,"output":270.0,"tier":{"size":272000}}]},"description":"More exact GPT-5.4 tier for demanding professional reasoning and agent tasks","name":"GPT-5.4 Pro"},"gpt-5.5":{"cost":{"cache_read":0.5,"input":5.0,"output":30.0,"tiers":[{"cache_read":1.0,"input":10.0,"output":45.0,"tier":{"size":272000}}]},"description":"Default frontier GPT for coding, computer use, research, and knowledge work","name":"GPT-5.5"},"gpt-5.5-pro":{"cost":{"input":30.0,"output":180.0,"tiers":[{"input":60.0,"output":270.0,"tier":{"size":272000}}]},"description":"Highest-accuracy GPT-5.5 tier for slower, precision-heavy reasoning and coding","name":"GPT-5.5 Pro"},"gpt-5.6":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5.0,"output":30.0,"tiers":[{"cache_read":1.0,"cache_write":12.5,"input":10.0,"output":45.0,"tier":{"size":272000}}]},"description":"Frontier GPT-5.6 model for complex professional work, coding, and agentic workflows","name":"GPT-5.6"},"gpt-5.6-luna":{"cost":{"cache_read":0.02,"cache_write":0.25,"input":0.2,"output":1.2,"tiers":[{"cache_read":0.04,"cache_write":0.5,"input":0.4,"output":1.8,"tier":{"size":272000}}]},"description":"Cost-efficient GPT-5.6 model for fast, high-volume workloads","name":"GPT-5.6 Luna"},"gpt-5.6-sol":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5.0,"output":30.0,"tiers":[{"cache_read":1.0,"cache_write":12.5,"input":10.0,"output":45.0,"tier":{"size":272000}}]},"description":"Frontier GPT-5.6 model for complex professional work, coding, and agentic workflows","name":"GPT-5.6 Sol"},"gpt-5.6-terra":{"cost":{"cache_read":0.2,"cache_write":2.5,"input":2.0,"output":12.0,"tiers":[{"cache_read":0.4,"cache_write":5.0,"input":4.0,"output":18.0,"tier":{"size":272000}}]},"description":"Balanced GPT-5.6 model for capable, cost-efficient everyday work","name":"GPT-5.6 Terra"},"gpt-image-1":{"description":"OpenAI image model for production generation, edits, and brand-safe visual workflows","name":"gpt-image-1"},"gpt-image-1-mini":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"gpt-image-1-mini"},"gpt-image-1.5":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"gpt-image-1.5"},"gpt-image-2":{"cost":{"cache_read":1.25,"input":5.0,"output":30.0},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"gpt-image-2"},"gpt-realtime-2.1":{"cost":{"cache_read":0.4,"input":4.0,"input_audio":32.0,"output":24.0,"output_audio":64.0},"description":"Realtime speech-to-speech model with configurable reasoning, tool use, and robust voice-agent behavior","name":"GPT-Realtime-2.1"},"o1":{"cost":{"cache_read":7.5,"input":15.0,"output":60.0},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"o1"},"o1-pro":{"cost":{"input":150.0,"output":600.0},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"o1-pro"},"o3":{"cost":{"cache_read":0.5,"input":2.0,"output":8.0},"description":"Deliberate o-series reasoner for hard math, coding, and multi-step analysis","name":"o3"},"o3-mini":{"cost":{"cache_read":0.55,"input":1.1,"output":4.4},"description":"Smaller o-series reasoner for economical coding, math, and planning tasks","name":"o3-mini"},"o3-pro":{"cost":{"input":20.0,"output":80.0},"description":"High-effort o3 tier for difficult technical reasoning and careful answers","name":"o3-pro"},"o4-mini":{"cost":{"cache_read":0.275,"input":1.1,"output":4.4},"description":"Fast o-series model for compact reasoning, coding, and tool use","name":"o4-mini"},"text-embedding-3-large":{"cost":{"input":0.13,"output":0.0},"description":"Embedding model for semantic search, retrieval, clustering, and ranking pipelines","name":"text-embedding-3-large"},"text-embedding-3-small":{"cost":{"input":0.02,"output":0.0},"description":"Embedding model for semantic search, retrieval, clustering, and ranking pipelines","name":"text-embedding-3-small"},"text-embedding-ada-002":{"cost":{"input":0.1,"output":0.0},"description":"Embedding model for semantic search, retrieval, clustering, and ranking pipelines","name":"text-embedding-ada-002"}}},"openrouter":{"models":{"ai21/jamba-large-1.7":{"cost":{"input":2,"output":8},"description":"Flagship model for demanding analysis, coding, and production agent workflows","name":"Jamba Large 1.7"},"aion-labs/aion-2.0":{"cost":{"cache_read":0.2,"input":0.8,"output":1.6},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"Aion-2.0"},"aion-labs/aion-3.0":{"cost":{"cache_read":0.75,"input":3,"output":6},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"Aion-3.0"},"aion-labs/aion-3.0-mini":{"cost":{"cache_read":0.18,"input":0.7,"output":1.4},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Aion-3.0-Mini"},"aion-labs/aion-rp-llama-3.1-8b":{"cost":{"input":0.8,"output":1.6},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Aion-RP 1.0 (8B)"},"allenai/olmo-3-32b-think":{"cost":{"input":0.15,"output":0.5},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"Olmo 3 32B Think"},"amazon/nova-2-lite-v1":{"cost":{"input":0.3,"output":2.5},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Nova 2 Lite"},"amazon/nova-lite-v1":{"cost":{"input":0.06,"output":0.24},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Nova Lite 1.0"},"amazon/nova-micro-v1":{"cost":{"input":0.035,"output":0.14},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Nova Micro 1.0"},"amazon/nova-premier-v1":{"cost":{"cache_read":0.625,"input":2.5,"output":12.5},"description":"Flagship model for demanding analysis, coding, and production agent workflows","name":"Nova Premier 1.0"},"amazon/nova-pro-v1":{"cost":{"input":0.8,"output":3.2},"description":"Flagship model for demanding analysis, coding, and production agent workflows","name":"Nova Pro 1.0"},"anthracite-org/magnum-v4-72b":{"cost":{"input":3,"output":5},"description":"Open-weight instruction model for adaptable chat and self-hosted production workloads","name":"Magnum v4 72B"},"anthropic/claude-3-haiku":{"cost":{"cache_read":0.03,"cache_write":0.3,"input":0.25,"output":1.25},"description":"Fast Claude model for responsive assistance, classification, and lightweight agents","name":"Claude 3 Haiku"},"anthropic/claude-fable-5":{"cost":{"cache_read":1,"cache_write":12.5,"input":10,"output":50},"description":"Claude model for creative writing, analysis, and controlled agent workflows","name":"Claude Fable 5"},"anthropic/claude-haiku-4.5":{"cost":{"cache_read":0.1,"cache_write":1.25,"input":1,"output":5},"description":"Fast Claude lane for lightweight agents, office tasks, and responsive chat","name":"Claude Haiku 4.5 (latest)"},"anthropic/claude-opus-4":{"cost":{"cache_read":1.5,"cache_write":18.75,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4"},"anthropic/claude-opus-4.1":{"cost":{"cache_read":1.5,"cache_write":18.75,"input":15,"output":75},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4.1 (latest)"},"anthropic/claude-opus-4.5":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4.5 (latest)"},"anthropic/claude-opus-4.6":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25,"tiers":[{"cache_read":1,"cache_write":12.5,"input":10,"output":37.5,"tier":{"size":200000,"type":"context"}}]},"description":"High-end Claude for difficult coding, planning, and slower expert reasoning","name":"Claude Opus 4.6"},"anthropic/claude-opus-4.7":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25,"tiers":[{"cache_read":1,"cache_write":12.5,"input":10,"output":37.5,"tier":{"size":200000,"type":"context"}}]},"description":"Stronger Opus tier for advanced software work and high-stakes reasoning","name":"Claude Opus 4.7"},"anthropic/claude-opus-4.7-fast":{"cost":{"cache_read":3,"cache_write":37.5,"input":30,"output":150},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4.7 (Fast)"},"anthropic/claude-opus-4.8":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Top Claude Opus tier for the hardest reasoning, coding, and long-horizon agents","name":"Claude Opus 4.8"},"anthropic/claude-opus-4.8-fast":{"cost":{"cache_read":1,"cache_write":12.5,"input":10,"output":50},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 4.8 (Fast)"},"anthropic/claude-opus-5":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 5"},"anthropic/claude-opus-5-fast":{"cost":{"cache_read":1,"cache_write":12.5,"input":10,"output":50},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus 5 (Fast)"},"anthropic/claude-sonnet-4":{"cost":{"cache_read":0.3,"cache_write":3.75,"input":3,"output":15,"tiers":[{"cache_read":0.6,"cache_write":7.5,"input":6,"output":22.5,"tier":{"size":200000,"type":"context"}}]},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude Sonnet 4"},"anthropic/claude-sonnet-4.5":{"cost":{"cache_read":0.3,"cache_write":3.75,"input":3,"output":15,"tiers":[{"cache_read":0.6,"cache_write":7.5,"input":6,"output":22.5,"tier":{"size":200000,"type":"context"}}]},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude Sonnet 4.5 (latest)"},"anthropic/claude-sonnet-4.6":{"cost":{"cache_read":0.3,"cache_write":3.75,"input":3,"output":15,"tiers":[{"cache_read":0.6,"cache_write":7.5,"input":6,"output":22.5,"tier":{"size":200000,"type":"context"}}]},"description":"Claude workhorse for coding agents, careful analysis, and production cost control","name":"Claude Sonnet 4.6"},"anthropic/claude-sonnet-5":{"cost":{"cache_read":0.2,"cache_write":2.5,"input":2,"output":10},"description":"Everyday Claude agent model for coding, planning, browsing, and general work","name":"Claude Sonnet 5"},"arcee-ai/trinity-large-thinking":{"cost":{"cache_read":0.06,"input":0.22,"output":0.85},"description":"Flagship model for demanding analysis, coding, and production agent workflows","name":"Trinity Large Thinking"},"arcee-ai/virtuoso-large":{"cost":{"input":0.75,"output":1.2},"description":"Flagship model for demanding analysis, coding, and production agent workflows","name":"Virtuoso Large"},"baidu/ernie-4.5-vl-424b-a47b":{"cost":{"input":0.42,"output":1.25},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"ERNIE 4.5 VL 424B A47B "},"bytedance-seed/seed-1.6":{"cost":{"input":0.25,"output":2,"tiers":[{"input":0.5,"output":4,"tier":{"size":128000,"type":"context"}}]},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Seed 1.6"},"bytedance-seed/seed-1.6-flash":{"cost":{"input":0.075,"output":0.3,"tiers":[{"input":0.1,"output":0.8,"tier":{"size":128000,"type":"context"}}]},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Seed 1.6 Flash"},"bytedance-seed/seed-2.0-lite":{"cost":{"input":0.25,"output":2,"tiers":[{"input":0.5,"output":4,"tier":{"size":128000,"type":"context"}}]},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Seed-2.0-Lite"},"bytedance-seed/seed-2.0-mini":{"cost":{"input":0.1,"output":0.4,"tiers":[{"input":0.2,"output":0.8,"tier":{"size":128000,"type":"context"}}]},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Seed-2.0-Mini"},"bytedance/ui-tars-1.5-7b":{"cost":{"cache_read":0.1,"input":0.1,"output":0.2},"description":"Multimodal model for analyzing text, images, documents, and rich media","name":"UI-TARS 7B "},"cognitivecomputations/dolphin-mistral-24b-venice-edition":{"cost":{"input":0.2,"output":0.9},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Uncensored"},"cohere/command-a":{"cost":{"input":2.5,"output":10},"description":"Cohere command model for multilingual enterprise agents, tools, and chat","name":"Command A"},"cohere/command-r-08-2024":{"cost":{"input":0.15,"output":0.6},"description":"Cohere retrieval model for long-context chat and enterprise RAG workflows","name":"Command R"},"cohere/command-r-plus-08-2024":{"cost":{"input":2.5,"output":10},"description":"Cohere's RAG workhorse for long-context enterprise search and tool use","name":"Command R+"},"cohere/command-r7b-12-2024":{"cost":{"input":0.0375,"output":0.15},"description":"Cohere retrieval model for long-context chat and enterprise RAG workflows","name":"Command R7B"},"cohere/north-mini-code:free":{"cost":{"input":0,"output":0},"description":"Cohere coding model for practical software engineering and agentic edits","name":"North Mini Code (free)"},"deepcogito/cogito-v2.1-671b":{"cost":{"input":1.25,"output":1.25},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"Cogito v2.1 671B"},"deepseek/deepseek-chat":{"cost":{"input":0.2574,"output":1.0287},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek Chat"},"deepseek/deepseek-chat-v3-0324":{"cost":{"cache_read":0.135,"input":0.27,"output":1.12},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek V3 0324"},"deepseek/deepseek-chat-v3.1":{"cost":{"cache_read":0.13,"input":0.25,"output":0.95},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek V3.1"},"deepseek/deepseek-r1":{"cost":{"input":0.7,"output":2.5},"description":"Classic open reasoning model for transparent math, coding, and deliberate problem solving","name":"DeepSeek-R1"},"deepseek/deepseek-r1-0528":{"cost":{"cache_read":0.35,"input":0.5,"output":2.15},"description":"DeepSeek reasoning model for multi-step analysis, math, coding, and tools","name":"R1 0528"},"deepseek/deepseek-r1-distill-llama-70b":{"cost":{"input":0.8,"output":0.8},"description":"DeepSeek reasoning model for multi-step analysis, math, coding, and tools","name":"R1 Distill Llama 70B"},"deepseek/deepseek-v3.1-terminus":{"cost":{"cache_read":0.135,"input":0.27,"output":1},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek V3.1 Terminus"},"deepseek/deepseek-v3.2":{"cost":{"cache_read":0.1345,"input":0.269,"output":0.4},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek V3.2"},"deepseek/deepseek-v3.2-exp":{"cost":{"input":0.27,"output":0.41},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek V3.2 Exp"},"deepseek/deepseek-v4-flash":{"cost":{"cache_read":0.028,"input":0.14,"output":0.28},"description":"Fast DeepSeek V4 lane for economical reasoning, coding, and long-context work","name":"DeepSeek V4 Flash"},"deepseek/deepseek-v4-flash-0731":{"cost":{"cache_read":0.018,"input":0.09,"output":0.18},"description":"Official DeepSeek V4 Flash release with enhanced agentic capabilities and integrated DSpark speculative decoding","name":"DeepSeek V4 Flash 0731"},"deepseek/deepseek-v4-pro":{"cost":{"cache_read":0.003625,"input":0.435,"output":0.87},"description":"Open MoE flagship with million-token context for coding and long agent runs","name":"DeepSeek V4 Pro"},"google/gemini-2.5-flash":{"cost":{"cache_read":0.03,"cache_write":0.083333,"input":0.3,"output":2.5,"reasoning":2.5},"description":"Fast Gemini workhorse for multimodal apps where latency and price matter","name":"Gemini 2.5 Flash"},"google/gemini-2.5-flash-image":{"cost":{"cache_read":0.03,"cache_write":0.083333,"input":0.3,"output":2.5},"description":"Nano Banana image model for fast generation, edits, and character-consistent assets","name":"Nano Banana"},"google/gemini-2.5-flash-lite":{"cost":{"cache_read":0.01,"cache_write":0.083333,"input":0.1,"output":0.4,"reasoning":0.4},"description":"Lean Gemini 2.5 lane for cheap multimodal traffic and quick agents","name":"Gemini 2.5 Flash-Lite"},"google/gemini-2.5-pro":{"cost":{"cache_read":0.125,"cache_write":0.375,"input":1.25,"output":10,"reasoning":10,"tiers":[{"cache_read":0.25,"input":2.5,"output":15,"tier":{"size":200000,"type":"context"}}]},"description":"Google's proven reasoning model for coding, math, and multimodal analysis","name":"Gemini 2.5 Pro"},"google/gemini-2.5-pro-preview":{"cost":{"cache_read":0.125,"cache_write":0.375,"input":1.25,"output":10,"reasoning":10,"tiers":[{"cache_read":0.25,"input":2.5,"output":15,"tier":{"size":200000,"type":"context"}}]},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Gemini 2.5 Pro Preview 06-05"},"google/gemini-2.5-pro-preview-05-06":{"cost":{"cache_read":0.125,"cache_write":0.375,"input":1.25,"output":10,"reasoning":10,"tiers":[{"cache_read":0.25,"input":2.5,"output":15,"tier":{"size":200000,"type":"context"}}]},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Gemini 2.5 Pro Preview 05-06"},"google/gemini-3-flash-preview":{"cost":{"cache_read":0.05,"cache_write":0.083333,"input":0.5,"output":3,"reasoning":3},"description":"New Gemini flash lane bringing frontier-style multimodal reasoning to cheaper runs","name":"Gemini 3 Flash Preview"},"google/gemini-3-pro-image":{"cost":{"cache_read":0.2,"cache_write":0.375,"input":2,"output":12,"reasoning":12},"description":"Nano Banana Pro for higher-fidelity image generation and design-heavy edits","name":"Nano Banana Pro"},"google/gemini-3-pro-image-preview":{"cost":{"cache_read":0.2,"cache_write":0.375,"input":2,"output":12,"reasoning":12},"description":"Nano Banana Pro for higher-fidelity image generation and design-heavy edits","name":"Nano Banana Pro"},"google/gemini-3.1-flash-image":{"cost":{"input":0.5,"output":3},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Nano Banana 2"},"google/gemini-3.1-flash-image-preview":{"cost":{"input":0.5,"output":3},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Nano Banana 2"},"google/gemini-3.1-flash-lite":{"cost":{"cache_read":0.025,"cache_write":0.083333,"input":0.25,"output":1.5,"reasoning":1.5},"description":"Low-latency Gemini model for high-volume multimodal and agent workloads","name":"Gemini 3.1 Flash Lite"},"google/gemini-3.1-flash-lite-image":{"cost":{"input":0.25,"output":1.5},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Nano Banana 2 Lite"},"google/gemini-3.1-flash-lite-preview":{"cost":{"cache_read":0.025,"cache_write":0.083333,"input":0.25,"output":1.5,"reasoning":1.5},"description":"Low-latency Gemini model for high-volume multimodal and agent workloads","name":"Gemini 3.1 Flash Lite Preview"},"google/gemini-3.1-pro-preview":{"cost":{"cache_read":0.2,"cache_write":0.375,"input":2,"output":12,"reasoning":12,"tiers":[{"cache_read":0.4,"input":4,"output":18,"tier":{"size":200000,"type":"context"}}]},"description":"Reasoning-first Gemini preview for agentic coding and complex problem solving","name":"Gemini 3.1 Pro Preview"},"google/gemini-3.1-pro-preview-customtools":{"cost":{"cache_read":0.2,"cache_write":0.375,"input":2,"output":12,"reasoning":12,"tiers":[{"cache_read":0.4,"input":4,"output":18,"tier":{"size":200000,"type":"context"}}]},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Gemini 3.1 Pro Preview Custom Tools"},"google/gemini-3.5-flash":{"cost":{"cache_read":0.15,"cache_write":0.083333,"input":1.5,"output":9,"reasoning":9},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini 3.5 Flash"},"google/gemini-3.5-flash-lite":{"cost":{"cache_read":0.03,"cache_write":0.083333,"input":0.3,"output":2.5,"reasoning":2.5},"description":"Low-latency Gemini model for high-volume multimodal and agent workloads","name":"Gemini 3.5 Flash Lite"},"google/gemini-3.6-flash":{"cost":{"cache_read":0.15,"cache_write":0.083333,"input":1.5,"output":7.5,"reasoning":7.5},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini 3.6 Flash"},"google/gemma-2-27b-it":{"cost":{"input":0.65,"output":0.65},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 2 27B"},"google/gemma-3-12b-it":{"cost":{"input":0.05,"output":0.15},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 3 12B"},"google/gemma-3-27b-it":{"cost":{"cache_read":0.04,"input":0.08,"output":0.45},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 3 27B"},"google/gemma-3-4b-it":{"cost":{"input":0.05,"output":0.1},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 3 4B"},"google/gemma-3n-e4b-it":{"cost":{"input":0.06,"output":0.12},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 3n 4B"},"google/gemma-4-26b-a4b-it":{"cost":{"input":0.07,"output":0.34},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 4 26B A4B IT"},"google/gemma-4-26b-a4b-it:free":{"cost":{"input":0,"output":0},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 4 26B A4B (free)"},"google/gemma-4-31b-it":{"cost":{"cache_read":0.1,"input":0.1,"output":0.34},"description":"Largest Gemma 4 instruction model for open, self-hosted chat and reasoning","name":"Gemma 4 31B IT"},"google/gemma-4-31b-it:free":{"cost":{"input":0,"output":0},"description":"Largest Gemma 4 instruction model for open, self-hosted chat and reasoning","name":"Gemma 4 31B (free)"},"google/lyria-3-clip-preview":{"cost":{"input":0,"output":0},"description":"Speech generation model for controllable voice, narration, and audio delivery","name":"Lyria 3 Clip Preview"},"google/lyria-3-pro-preview":{"cost":{"input":0,"output":0},"description":"Speech generation model for controllable voice, narration, and audio delivery","name":"Lyria 3 Pro Preview"},"gryphe/mythomax-l2-13b":{"cost":{"input":0.08,"output":0.11},"description":"Open-weight instruction model for adaptable chat and self-hosted production workloads","name":"MythoMax 13B"},"ibm-granite/granite-4.0-h-micro":{"cost":{"input":0.017,"output":0.112},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Granite 4.0 Micro"},"ibm-granite/granite-4.1-8b":{"cost":{"cache_read":0.05,"input":0.05,"output":0.1},"description":"Open-weight instruction model for adaptable chat and self-hosted production workloads","name":"Granite 4.1 8B"},"inception/mercury-2":{"cost":{"cache_read":0.025,"input":0.25,"output":0.75},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"Mercury 2"},"inclusionai/ling-2.6-1t":{"cost":{"cache_read":0.015,"input":0.075,"output":0.625},"description":"Tool-capable chat model for instruction following and agentic application workflows","name":"Ling-2.6-1T"},"inclusionai/ling-2.6-flash":{"cost":{"cache_read":0.002,"input":0.01,"output":0.03},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Ling-2.6-flash"},"inclusionai/ling-3.0-flash":{"cost":{"cache_read":0.0042,"input":0.021,"output":0.063},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Ling-3.0-flash"},"inclusionai/ling-3.0-tiny:free":{"cost":{"input":0,"output":0},"description":"Free provider route for experiments, demos, and cost-sensitive chat workloads","name":"Ling 3.0 Tiny (free)"},"inclusionai/ring-2.6-1t":{"cost":{"cache_read":0.015,"input":0.075,"output":0.625},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"Ring-2.6-1T"},"kwaipilot/kat-coder-air-v2.5":{"cost":{"cache_read":0.03,"input":0.15,"output":0.6},"description":"Coding model for repository understanding, refactors, and agentic engineering tasks","name":"KAT-Coder-Air V2.5"},"kwaipilot/kat-coder-pro-v2":{"cost":{"cache_read":0.06,"input":0.3,"output":1.2},"description":"Coding model for repository understanding, refactors, and agentic engineering tasks","name":"KAT-Coder-Pro V2"},"kwaipilot/kat-coder-pro-v2.5":{"cost":{"cache_read":0.15,"input":0.74,"output":2.96},"description":"Coding model for repository understanding, refactors, and agentic engineering tasks","name":"KAT-Coder-Pro V2.5"},"mancer/weaver":{"cost":{"input":0.5,"output":0.75},"description":"General-purpose chat model for instruction following, writing, and analysis","name":"Weaver (alpha)"},"meituan/longcat-2.0":{"cost":{"cache_read":0.006,"input":0.3,"output":1.2},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"LongCat 2.0"},"meta-llama/llama-3.1-70b-instruct":{"cost":{"input":0.4,"output":0.4},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.1 70B Instruct"},"meta-llama/llama-3.1-8b-instruct":{"cost":{"cache_read":0.025,"input":0.05,"output":0.08},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.1 8B Instruct"},"meta-llama/llama-3.2-1b-instruct":{"cost":{"input":0.027,"output":0.201},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.2 1B Instruct"},"meta-llama/llama-3.2-3b-instruct":{"cost":{"input":0.05,"output":0.33},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.2 3B Instruct"},"meta-llama/llama-3.3-70b-instruct":{"cost":{"input":0.1,"output":0.32},"description":"Popular open Llama workhorse for multilingual chat, coding, and self-hosting","name":"Llama-3.3-70B-Instruct"},"meta-llama/llama-4-maverick":{"cost":{"input":0.2,"output":0.8},"description":"Open multimodal Llama model for strong reasoning and fast responses","name":"Llama 4 Maverick"},"meta-llama/llama-4-scout":{"cost":{"input":0.1,"output":0.3},"description":"Open multimodal Llama model for long-context analysis and efficient agents","name":"Llama 4 Scout"},"meta-llama/llama-guard-4-12b":{"cost":{"input":0.18,"output":0.18},"description":"Safety model for policy screening, moderation, and risk-aware routing workflows","name":"Llama Guard 4 12B"},"meta/muse-spark-1.1":{"cost":{"cache_read":0.15,"input":1.25,"output":4.25},"description":"Open Llama multimodal model for image understanding and text reasoning","name":"Muse Spark 1.1"},"meta/muse-spark-1.2":{"cost":{"cache_read":0.15,"input":1.25,"output":4.25},"description":"Muse Spark 1.2 is a coding-focused update to Muse Spark 1.1 with improvements in code generation, complex debugging, codebase understanding, and end-to-end developer workflows.","name":"Muse Spark 1.2"},"microsoft/phi-4":{"cost":{"input":0.07,"output":0.14},"description":"Open-weight instruction model for adaptable chat and self-hosted production workloads","name":"Phi 4"},"microsoft/wizardlm-2-8x22b":{"cost":{"input":0.62,"output":0.62},"description":"Open-weight instruction model for adaptable chat and self-hosted production workloads","name":"WizardLM-2 8x22B"},"minimax/minimax-01":{"cost":{"input":0.2,"output":1.1},"description":"MiniMax multimodal coding model for long-context reasoning and agent tasks","name":"MiniMax-01"},"minimax/minimax-m1":{"cost":{"input":0.55,"output":2.2},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"MiniMax M1"},"minimax/minimax-m2":{"cost":{"input":0.255,"output":1.02},"description":"Efficient open MiniMax model built for coding agents and tool-heavy workflows","name":"MiniMax-M2"},"minimax/minimax-m2-her":{"cost":{"cache_read":0.03,"input":0.3,"output":1.2},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"MiniMax M2-her"},"minimax/minimax-m2.1":{"cost":{"cache_read":0.03,"input":0.3,"output":1.2},"description":"Earlier MiniMax agent model for practical coding and productivity tasks","name":"MiniMax-M2.1"},"minimax/minimax-m2.5":{"cost":{"cache_read":0.05,"input":0.22,"output":0.9},"description":"Prior MiniMax coding model for agent workflows, office edits, and automation","name":"MiniMax-M2.5"},"minimax/minimax-m2.7":{"cost":{"cache_read":0.06,"input":0.3,"output":1.2},"description":"Open MiniMax flagship for coding agents, office automation, and complex environments","name":"MiniMax-M2.7"},"minimax/minimax-m3":{"cost":{"cache_read":0.06,"input":0.3,"output":1.2},"description":"MiniMax multimodal model for long-context coding, perception, and agent planning","name":"MiniMax-M3"},"mistralai/codestral-2508":{"cost":{"cache_read":0.03,"input":0.3,"output":0.9},"description":"Mistral coding model for code completion, generation, and developer workflows","name":"Codestral 2508"},"mistralai/ministral-14b-2512":{"cost":{"cache_read":0.02,"input":0.2,"output":0.2},"description":"Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads","name":"Ministral 3 14B 2512"},"mistralai/ministral-3b-2512":{"cost":{"cache_read":0.01,"input":0.1,"output":0.1},"description":"Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads","name":"Ministral 3 3B 2512"},"mistralai/ministral-8b-2512":{"cost":{"cache_read":0.015,"input":0.15,"output":0.15},"description":"Compact Mistral model for edge, latency-sensitive, and cost-efficient workloads","name":"Ministral 3 8B 2512"},"mistralai/mistral-large":{"cost":{"cache_read":0.2,"input":2,"output":6},"description":"Flagship Mistral model for advanced reasoning, coding, and multilingual work","name":"Mistral Large"},"mistralai/mistral-large-2407":{"cost":{"cache_read":0.2,"input":2,"output":6},"description":"Flagship Mistral model for advanced reasoning, coding, and multilingual work","name":"Mistral Large 2407"},"mistralai/mistral-large-2512":{"cost":{"cache_read":0.05,"input":0.5,"output":1.5},"description":"Mistral's largest general model for enterprise agents, coding, and multilingual reasoning","name":"Mistral Large 3"},"mistralai/mistral-medium-3":{"cost":{"cache_read":0.04,"input":0.4,"output":2},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral Medium 3"},"mistralai/mistral-medium-3-5":{"cost":{"input":1.5,"output":7.5},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral Medium 3.5"},"mistralai/mistral-medium-3.1":{"cost":{"cache_read":0.04,"input":0.4,"output":2},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral Medium 3.1"},"mistralai/mistral-nemo":{"cost":{"input":0.019,"output":0.03},"description":"Efficient Mistral-NVIDIA open model for multilingual chat and local deployment","name":"Mistral Nemo"},"mistralai/mistral-saba":{"cost":{"cache_read":0.02,"input":0.2,"output":0.6},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Saba"},"mistralai/mistral-small-24b-instruct-2501":{"cost":{"input":0.05,"output":0.08},"description":"Efficient Mistral model for fast chat, extraction, and production assistants","name":"Mistral Small 3"},"mistralai/mistral-small-2603":{"cost":{"cache_read":0.015,"input":0.15,"output":0.6},"description":"Fast Mistral production model for chat, extraction, and cost-sensitive agents","name":"Mistral Small 4"},"mistralai/mistral-small-3.1-24b-instruct":{"cost":{"input":0.351,"output":0.555},"description":"Efficient Mistral model for fast chat, extraction, and production assistants","name":"Mistral Small 3.1 24B"},"mistralai/mistral-small-3.2-24b-instruct":{"cost":{"input":0.09375,"output":0.25},"description":"Efficient Mistral model for fast chat, extraction, and production assistants","name":"Mistral Small 3.2 24B"},"mistralai/mixtral-8x22b-instruct":{"cost":{"cache_read":0.2,"input":2,"output":6},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mixtral 8x22B Instruct"},"mistralai/voxtral-small-24b-2507":{"cost":{"cache_read":0.01,"input":0.1,"output":0.3},"description":"Efficient Mistral model for fast chat, extraction, and production assistants","name":"Voxtral Small 24B 2507"},"moonshotai/kimi-k2":{"cost":{"input":0.57,"output":2.3},"description":"Kimi model for long-context chat, coding, and agentic reasoning","name":"Kimi K2 0711"},"moonshotai/kimi-k2-0905":{"cost":{"input":0.6,"output":2.5},"description":"Kimi model for long-context chat, coding, and agentic reasoning","name":"Kimi K2 0905"},"moonshotai/kimi-k2-thinking":{"cost":{"cache_read":0.15,"input":0.6,"output":2.5},"description":"Thinking Kimi model for slower research passes, planning, and hard technical questions","name":"Kimi K2 Thinking"},"moonshotai/kimi-k2.5":{"cost":{"cache_read":0.095,"input":0.57,"output":2.85},"description":"Earlier Kimi frontier model for long-context agents, coding, and multimodal work","name":"Kimi K2.5"},"moonshotai/kimi-k2.6":{"cost":{"cache_read":0.0976,"input":0.5795,"output":2.44},"description":"Multimodal Kimi workhorse for agent loops, coding tasks, and visual context","name":"Kimi K2.6"},"moonshotai/kimi-k2.7-code":{"cost":{"cache_read":0.15,"input":0.7,"output":3.5},"description":"Coding-focused Kimi model, stronger on long-horizon repo work with less overthinking","name":"Kimi K2.7 Code"},"moonshotai/kimi-k3":{"cost":{"cache_read":0.3,"input":3,"output":15},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi K3"},"morph/morph-v3-fast":{"cost":{"input":0.8,"output":1.2},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Morph V3 Fast"},"morph/morph-v3-large":{"cost":{"input":0.9,"output":1.9},"description":"Flagship model for demanding analysis, coding, and production agent workflows","name":"Morph V3 Large"},"nex-agi/nex-n2-mini":{"cost":{"cache_read":0.0025,"input":0.025,"output":0.1},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Nex-N2-Mini"},"nex-agi/nex-n2-pro":{"cost":{"cache_read":0.025,"input":0.25,"output":1},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Nex-N2-Pro"},"nousresearch/hermes-3-llama-3.1-405b":{"cost":{"input":1,"output":1},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Hermes 3 405B Instruct"},"nousresearch/hermes-3-llama-3.1-70b":{"cost":{"input":0.7,"output":0.7},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Hermes 3 70B Instruct"},"nousresearch/hermes-4-405b":{"cost":{"input":1,"output":3},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"Hermes 4 405B"},"nousresearch/hermes-4-70b":{"cost":{"input":0.13,"output":0.4},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"Hermes 4 70B"},"nvidia/nemotron-3-nano-30b-a3b":{"cost":{"cache_read":0.03,"input":0.05,"output":0.2},"description":"Small Nemotron 3 MoE for efficient coding, math, and long-context agents","name":"Nemotron 3 Nano 30B A3B"},"nvidia/nemotron-3-nano-30b-a3b:free":{"cost":{"input":0,"output":0},"description":"Small Nemotron 3 MoE for efficient coding, math, and long-context agents","name":"Nemotron 3 Nano 30B A3B (free)"},"nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free":{"cost":{"input":0,"output":0},"description":"Open Nemotron omni model combining reasoning with text, vision, and audio","name":"Nemotron 3 Nano Omni (free)"},"nvidia/nemotron-3-super-120b-a12b":{"cost":{"input":0.085,"output":0.4},"description":"Nemotron middle tier for collaborative agents and high-volume reasoning workloads","name":"Nemotron 3 Super 120B A12B"},"nvidia/nemotron-3-super-120b-a12b:free":{"cost":{"input":0,"output":0},"description":"Nemotron middle tier for collaborative agents and high-volume reasoning workloads","name":"Nemotron 3 Super (free)"},"nvidia/nemotron-3-ultra-550b-a55b":{"cost":{"cache_read":0.2,"input":0.6,"output":3.6},"description":"Largest Nemotron 3 model for maximum open-weight reasoning and agent accuracy","name":"Nemotron 3 Ultra 550B A55B"},"nvidia/nemotron-3-ultra-550b-a55b:free":{"cost":{"input":0,"output":0},"description":"Largest Nemotron 3 model for maximum open-weight reasoning and agent accuracy","name":"Nemotron 3 Ultra (free)"},"nvidia/nemotron-3.5-content-safety:free":{"cost":{"input":0,"output":0},"description":"Safety model for policy screening, moderation, and risk-aware routing workflows","name":"Nemotron 3.5 Content Safety (free)"},"nvidia/nemotron-nano-12b-v2-vl:free":{"cost":{"input":0,"output":0},"description":"Nemotron multimodal model for visual reasoning and agentic AI workflows","name":"Nemotron Nano 12B 2 VL (free)"},"nvidia/nemotron-nano-9b-v2:free":{"cost":{"input":0,"output":0},"description":"Compact Nemotron model for efficient reasoning and deployable AI agents","name":"Nemotron Nano 9B V2 (free)"},"openai/gpt-3.5-turbo":{"cost":{"input":0.5,"output":1.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-3.5-turbo"},"openai/gpt-3.5-turbo-0613":{"cost":{"input":1,"output":2},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-3.5 Turbo (older v0613)"},"openai/gpt-3.5-turbo-16k":{"cost":{"input":3,"output":4},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-3.5 Turbo 16k"},"openai/gpt-3.5-turbo-instruct":{"cost":{"input":1.5,"output":2},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-3.5 Turbo Instruct"},"openai/gpt-4":{"cost":{"input":30,"output":60},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-4"},"openai/gpt-4-turbo":{"cost":{"input":10,"output":30},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-4 Turbo"},"openai/gpt-4-turbo-preview":{"cost":{"input":10,"output":30},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-4 Turbo Preview"},"openai/gpt-4.1":{"cost":{"cache_read":0.5,"input":2,"output":8},"description":"Long-lived GPT workhorse for coding, instruction following, and production apps","name":"GPT-4.1"},"openai/gpt-4.1-mini":{"cost":{"cache_read":0.1,"input":0.4,"output":1.6},"description":"Affordable GPT-4.1 lane for fast coding help and structured extraction","name":"GPT-4.1 mini"},"openai/gpt-4.1-nano":{"cost":{"cache_read":0.025,"input":0.1,"output":0.4},"description":"Tiny GPT-4.1 option for classification, routing, and very high-volume tasks","name":"GPT-4.1 nano"},"openai/gpt-4o":{"cost":{"cache_read":1.25,"input":2.5,"output":10},"description":"Omni-era GPT for multimodal chat, practical coding, and general assistants","name":"GPT-4o"},"openai/gpt-4o-2024-05-13":{"cost":{"input":5,"output":15},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-4o (2024-05-13)"},"openai/gpt-4o-2024-08-06":{"cost":{"cache_read":1.25,"input":2.5,"output":10},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-4o (2024-08-06)"},"openai/gpt-4o-2024-11-20":{"cost":{"cache_read":1.25,"input":2.5,"output":10},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-4o (2024-11-20)"},"openai/gpt-4o-mini":{"cost":{"cache_read":0.075,"input":0.15,"output":0.6},"description":"Small omni GPT for cheap multimodal assistance and production-scale traffic","name":"GPT-4o mini"},"openai/gpt-4o-mini-2024-07-18":{"cost":{"cache_read":0.075,"input":0.15,"output":0.6},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-4o-mini (2024-07-18)"},"openai/gpt-5":{"cost":{"cache_read":0.125,"input":1.25,"output":10},"description":"Original GPT-5 workhorse for reasoning, coding, writing, and tool workflows","name":"GPT-5"},"openai/gpt-5-image":{"cost":{"cache_read":1.25,"input":10,"output":10},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"GPT-5 Image"},"openai/gpt-5-image-mini":{"cost":{"cache_read":0.25,"input":2.5,"output":2},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"GPT-5 Image Mini"},"openai/gpt-5-mini":{"cost":{"cache_read":0.025,"input":0.25,"output":2},"description":"Small GPT-5 for responsive agents, coding help, and everyday automation","name":"GPT-5 Mini"},"openai/gpt-5-nano":{"cost":{"cache_read":0.005,"input":0.05,"output":0.4},"description":"Tiny GPT-5 lane for routing, extraction, classification, and bulk jobs","name":"GPT-5 Nano"},"openai/gpt-5-pro":{"cost":{"input":15,"output":120},"description":"Higher-accuracy GPT-5 tier for tough analysis, coding reviews, and planning","name":"GPT-5 Pro"},"openai/gpt-5.1":{"cost":{"cache_read":0.125,"input":1.25,"output":10},"description":"Sharper GPT-5 generation for coding, product work, and tool-assisted tasks","name":"GPT-5.1"},"openai/gpt-5.1-codex":{"cost":{"cache_read":0.13,"input":1.25,"output":10},"description":"Codex GPT for repository edits, code review, and practical software agents","name":"GPT-5.1 Codex"},"openai/gpt-5.1-codex-max":{"cost":{"cache_read":0.125,"input":1.25,"output":10},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.1 Codex Max"},"openai/gpt-5.1-codex-mini":{"cost":{"cache_read":0.03,"input":0.25,"output":2},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.1 Codex mini"},"openai/gpt-5.2":{"cost":{"cache_read":0.175,"input":1.75,"output":14},"description":"Reliable GPT generation for broad coding, writing, and tool-assisted product work","name":"GPT-5.2"},"openai/gpt-5.2-chat":{"cost":{"cache_read":0.175,"input":1.75,"output":14},"description":"Chat-tuned GPT model for conversational assistance, writing, and tool workflows","name":"GPT-5.2 Chat"},"openai/gpt-5.2-codex":{"cost":{"cache_read":0.175,"input":1.75,"output":14},"description":"Code-specialist GPT for repository edits, reviews, and long-running software agents","name":"GPT-5.2 Codex"},"openai/gpt-5.2-pro":{"cost":{"input":21,"output":168},"description":"Higher-accuracy GPT-5.2 variant for tougher reasoning and review workflows","name":"GPT-5.2 Pro"},"openai/gpt-5.3-chat":{"cost":{"cache_read":0.175,"input":1.75,"output":14},"description":"Chat-tuned GPT model for conversational assistance, writing, and tool workflows","name":"GPT-5.3 Chat"},"openai/gpt-5.3-codex":{"cost":{"cache_read":0.175,"input":1.75,"output":14},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.3 Codex"},"openai/gpt-5.4":{"cost":{"cache_read":0.25,"input":2.5,"output":15,"tiers":[{"cache_read":0.5,"input":5,"output":22.5,"tier":{"size":272000,"type":"context"}}]},"description":"Agent-ready GPT for coding and computer-use workflows at a lower cost","name":"GPT-5.4"},"openai/gpt-5.4-image-2":{"cost":{"cache_read":2,"input":8,"output":15},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"GPT-5.4 Image 2"},"openai/gpt-5.4-mini":{"cost":{"cache_read":0.075,"input":0.75,"output":4.5},"description":"Strong small GPT for coding subagents, quick tool use, and high-volume work","name":"GPT-5.4 mini"},"openai/gpt-5.4-nano":{"cost":{"cache_read":0.02,"input":0.2,"output":1.25},"description":"Cheapest GPT-5.4 lane for simple routing, extraction, and bulk automation","name":"GPT-5.4 nano"},"openai/gpt-5.4-pro":{"cost":{"input":30,"output":180,"tiers":[{"input":60,"output":270,"tier":{"size":272000,"type":"context"}}]},"description":"More exact GPT-5.4 tier for demanding professional reasoning and agent tasks","name":"GPT-5.4 Pro"},"openai/gpt-5.5":{"cost":{"cache_read":0.5,"input":5,"output":30,"tiers":[{"cache_read":1,"input":10,"output":45,"tier":{"size":272000,"type":"context"}}]},"description":"Default frontier GPT for coding, computer use, research, and knowledge work","name":"GPT-5.5"},"openai/gpt-5.5-pro":{"cost":{"input":30,"output":180,"tiers":[{"input":60,"output":270,"tier":{"size":272000,"type":"context"}}]},"description":"Highest-accuracy GPT-5.5 tier for slower, precision-heavy reasoning and coding","name":"GPT-5.5 Pro"},"openai/gpt-5.6-luna":{"cost":{"cache_read":0.01,"cache_write":0.125,"input":0.1,"output":0.6,"tiers":[{"cache_read":0.02,"cache_write":0.25,"input":0.2,"output":0.9,"tier":{"size":272000,"type":"context"}}]},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-5.6 Luna"},"openai/gpt-5.6-luna-pro":{"cost":{"cache_read":0.01,"cache_write":0.125,"input":0.1,"output":0.6,"tiers":[{"cache_read":0.02,"cache_write":0.25,"input":0.2,"output":0.9,"tier":{"size":272000,"type":"context"}}]},"description":"Frontier GPT model for professional reasoning, coding, and multimodal work","name":"GPT-5.6 Luna Pro"},"openai/gpt-5.6-sol":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":30,"tiers":[{"cache_read":1,"cache_write":12.5,"input":10,"output":45,"tier":{"size":272000,"type":"context"}}]},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-5.6 Sol"},"openai/gpt-5.6-sol-pro":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":30,"tiers":[{"cache_read":1,"cache_write":12.5,"input":10,"output":45,"tier":{"size":272000,"type":"context"}}]},"description":"Frontier GPT model for professional reasoning, coding, and multimodal work","name":"GPT-5.6 Sol Pro"},"openai/gpt-5.6-terra":{"cost":{"cache_read":0.1,"cache_write":1.25,"input":1,"output":6,"tiers":[{"cache_read":0.2,"cache_write":2.5,"input":2,"output":9,"tier":{"size":272000,"type":"context"}}]},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-5.6 Terra"},"openai/gpt-5.6-terra-pro":{"cost":{"cache_read":0.1,"cache_write":1.25,"input":1,"output":6,"tiers":[{"cache_read":0.2,"cache_write":2.5,"input":2,"output":9,"tier":{"size":272000,"type":"context"}}]},"description":"Frontier GPT model for professional reasoning, coding, and multimodal work","name":"GPT-5.6 Terra Pro"},"openai/gpt-audio":{"cost":{"input":2.5,"output":10},"description":"Speech generation model for controllable voice, narration, and audio delivery","name":"GPT Audio"},"openai/gpt-audio-mini":{"cost":{"input":0.6,"output":2.4},"description":"Speech generation model for controllable voice, narration, and audio delivery","name":"GPT Audio Mini"},"openai/gpt-chat-latest":{"cost":{"cache_read":0.5,"input":5,"output":30},"description":"Chat-tuned GPT model for conversational assistance, writing, and tool workflows","name":"GPT Chat Latest"},"openai/gpt-oss-120b":{"cost":{"input":0.037,"output":0.17},"description":"Open GPT reasoning model for self-hosted agents and controllable deployments","name":"GPT OSS 120B"},"openai/gpt-oss-20b":{"cost":{"cache_read":0.03,"input":0.03,"output":0.13},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"GPT OSS 20B"},"openai/gpt-oss-20b:free":{"cost":{"input":0,"output":0},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"gpt-oss-20b (free)"},"openai/gpt-oss-safeguard-20b":{"cost":{"cache_read":0.0375,"input":0.075,"output":0.3},"description":"Safety model for policy screening, moderation, and risk-aware routing workflows","name":"gpt-oss-safeguard-20b"},"openai/o1":{"cost":{"cache_read":7.5,"input":15,"output":60},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"o1"},"openai/o1-pro":{"cost":{"input":150,"output":600},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"o1-pro"},"openai/o3":{"cost":{"cache_read":0.5,"input":2,"output":8},"description":"Deliberate o-series reasoner for hard math, coding, and multi-step analysis","name":"o3"},"openai/o3-mini":{"cost":{"cache_read":0.55,"input":1.1,"output":4.4},"description":"Smaller o-series reasoner for economical coding, math, and planning tasks","name":"o3-mini"},"openai/o3-mini-high":{"cost":{"cache_read":0.55,"input":1.1,"output":4.4},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"o3 Mini High"},"openai/o3-pro":{"cost":{"input":20,"output":80},"description":"High-effort o3 tier for difficult technical reasoning and careful answers","name":"o3-pro"},"openai/o4-mini":{"cost":{"cache_read":0.275,"input":1.1,"output":4.4},"description":"Fast o-series model for compact reasoning, coding, and tool use","name":"o4-mini"},"openai/o4-mini-high":{"cost":{"cache_read":0.275,"input":1.1,"output":4.4},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"o4 Mini High"},"openrouter/auto":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Auto Router"},"openrouter/bodybuilder":{"description":"Preview model for early access evaluation, prototyping, and compatibility testing","name":"Body Builder (beta)"},"openrouter/free":{"cost":{"input":0,"output":0},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Free Models Router"},"openrouter/fusion":{"description":"General-purpose chat model for instruction following, writing, and analysis","name":"Fusion"},"openrouter/pareto-code":{"description":"Coding model for repository understanding, refactors, and agentic engineering tasks","name":"Pareto Code Router"},"perceptron/perceptron-mk1":{"cost":{"input":0.15,"output":1.5},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Perceptron Mk1"},"perplexity/sonar":{"cost":{"input":1,"output":1},"description":"Sonar search model for current answers, retrieval, and citation-backed chat","name":"Sonar"},"perplexity/sonar-deep-research":{"cost":{"input":2,"output":8,"reasoning":3},"description":"Sonar search model for current answers, retrieval, and citation-backed chat","name":"Sonar Deep Research"},"perplexity/sonar-pro":{"cost":{"input":3,"output":15},"description":"Advanced Sonar search model for deeper research and cited synthesis","name":"Sonar Pro"},"perplexity/sonar-pro-search":{"cost":{"input":3,"output":15},"description":"Advanced Sonar search model for deeper research and cited synthesis","name":"Sonar Pro Search"},"perplexity/sonar-reasoning-pro":{"cost":{"input":2,"output":8},"description":"Web-grounded reasoning model for multi-step research and cited answers","name":"Sonar Reasoning Pro"},"poolside/laguna-s-2.1":{"cost":{"cache_read":0.009,"input":0.09,"output":0.18},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"Laguna S 2.1"},"poolside/laguna-s-2.1:free":{"cost":{"input":0,"output":0},"description":"Free provider route for experiments, demos, and cost-sensitive chat workloads","name":"Laguna S 2.1 (free)"},"poolside/laguna-xs-2.1":{"cost":{"cache_read":0.03,"input":0.06,"output":0.12},"description":"Agentic coding model from Poolside in the XS size class for local deployment","name":"Laguna XS 2.1"},"poolside/laguna-xs-2.1:free":{"cost":{"input":0,"output":0},"description":"Free provider route for experiments, demos, and cost-sensitive chat workloads","name":"Laguna XS 2.1 (free)"},"qwen/qwen-2.5-72b-instruct":{"cost":{"input":0.36,"output":0.4},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen2.5 72B Instruct"},"qwen/qwen-2.5-7b-instruct":{"cost":{"input":0.1,"output":0.2},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen2.5 7B Instruct"},"qwen/qwen-2.5-coder-32b-instruct":{"cost":{"input":0.66,"output":1},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen2.5 Coder 32B Instruct"},"qwen/qwen-plus":{"cost":{"cache_read":0.052,"cache_write":0.325,"input":0.26,"output":0.78,"tiers":[{"cache_read":0.156,"cache_write":0.975,"input":0.78,"output":2.34,"tier":{"size":256000,"type":"context"}}]},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen Plus"},"qwen/qwen-plus-2025-07-28":{"cost":{"input":0.26,"output":0.78,"tiers":[{"input":0.78,"output":2.34,"tier":{"size":256000,"type":"context"}}]},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen Plus 0728"},"qwen/qwen-plus-2025-07-28:thinking":{"cost":{"cache_write":0.5,"input":0.4,"output":1.2,"tiers":[{"cache_write":1.5,"input":1.2,"output":3.6,"tier":{"size":256000,"type":"context"}}]},"description":"Qwen reasoning model for deliberate problem solving, math, and coding","name":"Qwen Plus 0728 (thinking)"},"qwen/qwen2.5-vl-72b-instruct":{"cost":{"input":0.25,"output":0.75},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen2.5 VL 72B Instruct"},"qwen/qwen3-14b":{"cost":{"input":0.2275,"output":0.91},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3 14B"},"qwen/qwen3-235b-a22b":{"cost":{"input":0.455,"output":1.82},"description":"Large open Qwen MoE for multilingual reasoning, coding, and tool use","name":"Qwen3 235B-A22B"},"qwen/qwen3-235b-a22b-2507":{"cost":{"input":0.09,"output":0.55},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3 235B A22B Instruct 2507"},"qwen/qwen3-235b-a22b-thinking-2507":{"cost":{"input":0.23,"output":2.3},"description":"Qwen reasoning model for deliberate problem solving, math, and coding","name":"Qwen3 235B A22B Thinking 2507"},"qwen/qwen3-30b-a3b":{"cost":{"input":0.12,"output":0.5},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3 30B A3B"},"qwen/qwen3-30b-a3b-instruct-2507":{"cost":{"input":0.04815,"output":0.19305},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3 30B A3B Instruct 2507"},"qwen/qwen3-30b-a3b-thinking-2507":{"cost":{"input":0.2,"output":2.4},"description":"Qwen reasoning model for deliberate problem solving, math, and coding","name":"Qwen3 30B A3B Thinking 2507"},"qwen/qwen3-32b":{"cost":{"input":0.08,"output":0.28},"description":"Dense open Qwen model for self-hosted chat, reasoning, and coding","name":"Qwen3 32B"},"qwen/qwen3-8b":{"cost":{"input":0.117,"output":0.455},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3 8B"},"qwen/qwen3-coder":{"cost":{"cache_read":0.1,"input":0.3,"output":1},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen3 Coder 480B A35B"},"qwen/qwen3-coder-30b-a3b-instruct":{"cost":{"input":0.07,"output":0.27},"description":"Smaller Qwen coder for efficient local agents and repo-level fixes","name":"Qwen3-Coder 30B-A3B Instruct"},"qwen/qwen3-coder-flash":{"cost":{"cache_read":0.039,"cache_write":0.24375,"input":0.195,"output":0.975,"tiers":[{"cache_read":0.065,"cache_write":0.40625,"input":0.325,"output":1.625,"tier":{"size":32000,"type":"context"}},{"cache_read":0.104,"cache_write":0.65,"input":0.52,"output":2.6,"tier":{"size":128000,"type":"context"}}]},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen3 Coder Flash"},"qwen/qwen3-coder-next":{"cost":{"cache_read":0.07,"input":0.12,"output":0.8},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen3 Coder Next"},"qwen/qwen3-coder-plus":{"cost":{"cache_read":0.13,"cache_write":0.8125,"input":0.65,"output":3.25,"tiers":[{"cache_read":0.234,"cache_write":1.4625,"input":1.17,"output":5.85,"tier":{"size":32000,"type":"context"}},{"cache_read":0.39,"cache_write":2.4375,"input":1.95,"output":9.75,"tier":{"size":128000,"type":"context"}}]},"description":"Hosted Qwen coder for software agents, repo edits, and long-context code","name":"Qwen3 Coder Plus"},"qwen/qwen3-max":{"cost":{"cache_read":0.156,"cache_write":0.975,"input":0.78,"output":3.9,"tiers":[{"cache_read":0.312,"cache_write":1.95,"input":1.56,"output":7.8,"tier":{"size":32000,"type":"context"}},{"cache_read":0.39,"cache_write":2.4375,"input":1.95,"output":9.75,"tier":{"size":128000,"type":"context"}}]},"description":"Flagship Qwen3 model for coding agents, complex reasoning, and tool use","name":"Qwen3 Max"},"qwen/qwen3-max-thinking":{"cost":{"input":0.78,"output":3.9,"tiers":[{"input":1.56,"output":7.8,"tier":{"size":32000,"type":"context"}},{"input":1.95,"output":9.75,"tier":{"size":128000,"type":"context"}}]},"description":"Qwen reasoning model for deliberate problem solving, math, and coding","name":"Qwen3 Max Thinking"},"qwen/qwen3-next-80b-a3b-instruct":{"cost":{"input":0.09,"output":1.1},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3-Next 80B-A3B Instruct"},"qwen/qwen3-next-80b-a3b-thinking":{"cost":{"input":0.15,"output":1.2},"description":"Efficient Qwen thinking model for local reasoning, math, and coding agents","name":"Qwen3-Next 80B-A3B (Thinking)"},"qwen/qwen3-vl-235b-a22b-instruct":{"cost":{"cache_read":0.1,"input":0.21,"output":1.9},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3 VL 235B A22B Instruct"},"qwen/qwen3-vl-235b-a22b-thinking":{"cost":{"input":0.4,"output":4},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3 VL 235B A22B Thinking"},"qwen/qwen3-vl-30b-a3b-instruct":{"cost":{"input":0.15,"output":0.6},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3 VL 30B A3B Instruct"},"qwen/qwen3-vl-30b-a3b-thinking":{"cost":{"input":0.2,"output":2.4},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3 VL 30B A3B Thinking"},"qwen/qwen3-vl-32b-instruct":{"cost":{"input":0.104,"output":0.416},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3 VL 32B Instruct"},"qwen/qwen3-vl-8b-instruct":{"cost":{"input":0.117,"output":0.455},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3 VL 8B Instruct"},"qwen/qwen3-vl-8b-thinking":{"cost":{"input":0.18,"output":2.1},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3 VL 8B Thinking"},"qwen/qwen3.5-122b-a10b":{"cost":{"input":0.29,"output":2.4},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 122B-A10B"},"qwen/qwen3.5-27b":{"cost":{"input":0.195,"output":1.56},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 27B"},"qwen/qwen3.5-35b-a3b":{"cost":{"input":0.14,"output":1},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 35B-A3B"},"qwen/qwen3.5-397b-a17b":{"cost":{"input":0.39,"output":2.34},"description":"Large open Qwen multimodal MoE for visual agents and long technical tasks","name":"Qwen3.5 397B-A17B"},"qwen/qwen3.5-9b":{"cost":{"input":0.1,"output":0.15},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen3.5 9B"},"qwen/qwen3.5-flash-02-23":{"cost":{"input":0.065,"output":0.26},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5-Flash"},"qwen/qwen3.5-plus-02-15":{"cost":{"input":0.26,"output":1.56,"tiers":[{"input":0.325,"output":1.95,"tier":{"size":256000,"type":"context"}}]},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 Plus 2026-02-15"},"qwen/qwen3.5-plus-20260420":{"cost":{"cache_write":0.375,"input":0.3,"output":1.8,"tiers":[{"cache_write":0.46875,"input":0.375,"output":2.25,"tier":{"size":256000,"type":"context"}}]},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 Plus 2026-04-20"},"qwen/qwen3.6-27b":{"cost":{"cache_read":0.12,"input":0.6,"output":3.6},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.6 27B"},"qwen/qwen3.6-35b-a3b":{"cost":{"cache_read":0.05,"input":0.15,"output":1},"description":"Open multimodal Qwen MoE for local agents that need vision, audio, and code","name":"Qwen3.6 35B-A3B"},"qwen/qwen3.6-flash":{"cost":{"cache_write":0.234375,"input":0.1875,"output":1.125,"tiers":[{"cache_write":0.9375,"input":0.75,"output":3,"tier":{"size":256000,"type":"context"}}]},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.6 Flash"},"qwen/qwen3.6-max-preview":{"cost":{"cache_write":1.28375,"input":1.027,"output":6.162,"tiers":[{"cache_write":1.975,"input":1.58,"output":9.48,"tier":{"size":128000,"type":"context"}}]},"description":"Flagship Qwen model for complex reasoning, coding, and agentic workflows","name":"Qwen3.6 Max Preview"},"qwen/qwen3.6-plus":{"cost":{"cache_write":0.40625,"input":0.325,"output":1.95,"tiers":[{"cache_write":1.625,"input":1.3,"output":3.9,"tier":{"size":256000,"type":"context"}}]},"description":"Earlier Qwen multimodal workhorse for million-token agent and document tasks","name":"Qwen3.6 Plus"},"qwen/qwen3.7-flash":{"cost":{"cache_read":0.006,"cache_write":0.038,"input":0.03,"output":0.13,"tiers":[{"cache_read":0.02,"cache_write":0.125,"input":0.1,"output":0.4,"tier":{"size":32000,"type":"context"}},{"cache_read":0.04,"cache_write":0.25,"input":0.2,"output":0.8,"tier":{"size":256000,"type":"context"}}]},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.7 Flash"},"qwen/qwen3.7-max":{"cost":{"cache_read":0.295,"cache_write":1.84375,"input":1.475,"output":4.425},"description":"Qwen frontier model tuned for agent frameworks, coding assistants, and long tasks","name":"Qwen3.7 Max"},"qwen/qwen3.7-plus":{"cost":{"cache_read":0.064,"cache_write":0.4,"input":0.32,"output":1.28,"tiers":[{"cache_read":0.192,"cache_write":1.2,"input":0.96,"output":3.84,"tier":{"size":256000,"type":"context"}}]},"description":"Multimodal Qwen workhorse for long-context agents, visual inputs, and coding","name":"Qwen3.7 Plus"},"qwen/qwen3.8-max":{"cost":{"cache_read":0.25,"cache_write":2.5,"input":2,"output":6},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.8 Max"},"rekaai/reka-edge":{"cost":{"input":0.1,"output":0.1},"description":"Multimodal model for analyzing text, images, documents, and rich media","name":"Reka Edge"},"rekaai/reka-flash-3":{"cost":{"input":0.1,"output":0.2},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"Reka Flash 3"},"relace/relace-apply-3":{"cost":{"input":0.85,"output":1.25},"description":"General-purpose chat model for instruction following, writing, and analysis","name":"Relace Apply 3"},"relace/relace-search":{"cost":{"input":1,"output":3},"description":"Tool-capable chat model for instruction following and agentic application workflows","name":"Relace Search"},"sakana/fugu-ultra":{"cost":{"cache_read":0.5,"input":5,"output":30,"tiers":[{"cache_read":1,"input":10,"output":45,"tier":{"size":272000,"type":"context"}}]},"description":"Quality-first multi-agent model for hard research, analysis, and competitions","name":"Fugu Ultra"},"sao10k/l3-lunaris-8b":{"cost":{"input":0.04,"output":0.05},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3 8B Lunaris"},"sao10k/l3.1-euryale-70b":{"cost":{"input":0.85,"output":0.85},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.1 Euryale 70B v2.2"},"sao10k/l3.3-euryale-70b":{"cost":{"input":0.65,"output":0.75},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama 3.3 Euryale 70B"},"stepfun/step-3.5-flash":{"cost":{"input":0.1,"output":0.3},"description":"StepFun flash lane for quick multimodal reasoning and coding assistance","name":"Step 3.5 Flash"},"stepfun/step-3.7-flash":{"cost":{"cache_read":0.04,"input":0.2,"output":1.15},"description":"Newer StepFun flash model for faster agents, coding, and multimodal prompts","name":"Step 3.7 Flash"},"tencent/hunyuan-a13b-instruct":{"cost":{"input":0.14,"output":0.57},"description":"Tencent Hy reasoning model for coding, instruction following, and agent tasks","name":"Hunyuan A13B Instruct"},"tencent/hy3":{"cost":{"cache_read":0.033,"input":0.132,"output":0.528},"description":"Tencent Hy reasoning model for coding, instruction following, and agent tasks","name":"Hy3"},"tencent/hy3-preview":{"cost":{"cache_read":0.021,"input":0.063,"output":0.21},"description":"Tencent Hy reasoning model for coding, instruction following, and agent tasks","name":"Hy3 preview"},"thedrummer/cydonia-24b-v4.1":{"cost":{"cache_read":0.15,"input":0.3,"output":0.5},"description":"Open-weight instruction model for adaptable chat and self-hosted production workloads","name":"Cydonia 24B V4.1"},"thedrummer/rocinante-12b":{"cost":{"input":0.25,"output":0.5},"description":"Open-weight instruction model for adaptable chat and self-hosted production workloads","name":"Rocinante 12B"},"thedrummer/skyfall-36b-v2":{"cost":{"cache_read":0.25,"input":0.55,"output":0.8},"description":"Open-weight instruction model for adaptable chat and self-hosted production workloads","name":"Skyfall 36B V2"},"thedrummer/unslopnemo-12b":{"cost":{"input":0.4,"output":0.4},"description":"Open-weight instruction model for adaptable chat and self-hosted production workloads","name":"UnslopNemo 12B"},"thinkingmachines/inkling":{"cost":{"cache_read":0.16,"input":0.95,"output":4.05},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Inkling"},"thinkingmachines/inkling-small":{"cost":{"cache_read":0.1,"input":0.45,"output":1.2},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Inkling Small"},"undi95/remm-slerp-l2-13b":{"cost":{"input":0.45,"output":0.65},"description":"Open-weight instruction model for adaptable chat and self-hosted production workloads","name":"ReMM SLERP 13B"},"upstage/solar-pro-3":{"cost":{"cache_read":0.015,"input":0.15,"output":0.6},"description":"Flagship model for demanding analysis, coding, and production agent workflows","name":"Solar Pro 3"},"writer/palmyra-x5":{"cost":{"input":0.6,"output":6},"description":"General-purpose chat model for instruction following, writing, and analysis","name":"Palmyra X5"},"x-ai/grok-4.20":{"cost":{"cache_read":0.2,"input":1.25,"output":2.5,"tiers":[{"cache_read":0.4,"input":2.5,"output":5,"tier":{"size":200000,"type":"context"}}]},"description":"Grok model for agentic tool use, reasoning, coding, and live assistance","name":"Grok 4.20"},"x-ai/grok-4.20-multi-agent":{"cost":{"cache_read":0.2,"input":1.25,"output":2.5,"tiers":[{"cache_read":0.4,"input":2.5,"output":5,"tier":{"size":200000,"type":"context"}}]},"description":"Grok model for agentic tool use, reasoning, coding, and live assistance","name":"Grok 4.20 Multi-Agent"},"x-ai/grok-4.3":{"cost":{"cache_read":0.2,"input":1.25,"output":2.5,"tiers":[{"cache_read":0.4,"input":2.5,"output":5,"tier":{"size":200000,"type":"context"}}]},"description":"xAI's default Grok for chat, coding, agentic tools, and lower hallucination risk","name":"Grok 4.3"},"x-ai/grok-4.5":{"cost":{"cache_read":0.3,"input":2,"output":6,"tiers":[{"cache_read":0.6,"input":4,"output":12,"tier":{"size":200000,"type":"context"}}]},"description":"xAI's latest Grok for chat, coding, agentic tools, and lower hallucination risk","name":"Grok 4.5"},"x-ai/grok-build-0.1":{"cost":{"cache_read":0.2,"input":1,"output":2,"tiers":[{"cache_read":0.4,"input":2,"output":4,"tier":{"size":200000,"type":"context"}}]},"description":"Fast Grok coding model tuned for agentic engineering and iterative edits","name":"Grok Build 0.1"},"xiaomi/mimo-v2.5":{"cost":{"cache_read":0.0028,"input":0.14,"output":0.28},"description":"Open MiMo model for multimodal coding agents and long-context automation","name":"MiMo-V2.5"},"xiaomi/mimo-v2.5-pro":{"cost":{"cache_read":0.0036,"input":0.435,"output":0.87},"description":"Stronger MiMo Pro tier for multimodal reasoning and coding-agent execution","name":"MiMo-V2.5-Pro"},"z-ai/glm-4.5":{"cost":{"cache_read":0.11,"input":0.6,"output":2.2},"description":"Hybrid-reasoning GLM release that made the 4.5 line broadly useful","name":"GLM-4.5"},"z-ai/glm-4.5-air":{"cost":{"cache_read":0.025,"input":0.13,"output":0.85},"description":"Lighter GLM-4.5 variant for fast coding assistance and cheaper agents","name":"GLM-4.5-Air"},"z-ai/glm-4.5v":{"cost":{"cache_read":0.11,"input":0.6,"output":1.8},"description":"GLM vision model for visual reasoning, documents, and multimodal agents","name":"GLM-4.5V"},"z-ai/glm-4.6":{"cost":{"cache_read":0.1,"input":0.5,"output":2},"description":"Late GLM-4 workhorse for coding agents, reasoning, and structured tasks","name":"GLM-4.6"},"z-ai/glm-4.6v":{"cost":{"cache_read":0.055,"input":0.3,"output":0.9},"description":"GLM vision model for visual reasoning, documents, and multimodal agents","name":"GLM-4.6V"},"z-ai/glm-4.7":{"cost":{"cache_read":0.08,"input":0.4,"output":1.75},"description":"Mature GLM model for dependable coding, reasoning, and structured agent tasks","name":"GLM-4.7"},"z-ai/glm-4.7-flash":{"cost":{"cache_read":0.01,"input":0.06,"output":0.4},"description":"Budget GLM lane for fast coding help, routing, and everyday automation","name":"GLM-4.7-Flash"},"z-ai/glm-5":{"cost":{"cache_read":0.2,"input":0.95,"output":2.55},"description":"General GLM flagship for coding, analysis, and tool-heavy engineering workflows","name":"GLM-5"},"z-ai/glm-5-turbo":{"cost":{"cache_read":0.24,"input":1.2,"output":4},"description":"Faster GLM-5 lane for coding agents that need lower latency","name":"GLM-5-Turbo"},"z-ai/glm-5.1":{"cost":{"cache_read":0.1768,"input":0.952,"output":2.992},"description":"Strong GLM coding model for agentic engineering, terminals, and repository generation","name":"GLM-5.1"},"z-ai/glm-5.2":{"cost":{"cache_read":0.03822,"input":0.2058,"output":0.6468},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM-5.2"},"z-ai/glm-5v-turbo":{"cost":{"cache_read":0.24,"input":1.2,"output":4},"description":"Fast GLM vision model for screenshots, documents, and multimodal agent tasks","name":"GLM-5V-Turbo"},"~anthropic/claude-fable-latest":{"cost":{"cache_read":1,"cache_write":12.5,"input":10,"output":50},"description":"Claude model for creative writing, analysis, and controlled agent workflows","name":"Claude Fable Latest"},"~anthropic/claude-haiku-latest":{"cost":{"cache_read":0.1,"cache_write":1.25,"input":1,"output":5},"description":"Fast Claude model for responsive assistance, classification, and lightweight agents","name":"Anthropic Claude Haiku Latest"},"~anthropic/claude-opus-latest":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":25},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude Opus Latest"},"~anthropic/claude-sonnet-latest":{"cost":{"cache_read":0.2,"cache_write":2.5,"input":2,"output":10},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Anthropic Claude Sonnet Latest"},"~deepseek/deepseek-v4-flash-latest":{"cost":{"cache_read":0.01792,"input":0.0896,"output":0.1792},"description":"Fast DeepSeek model for efficient chat, coding help, and agent loops","name":"DeepSeek V4 Flash Latest"},"~google/gemini-flash-latest":{"cost":{"cache_read":0.15,"cache_write":0.083333,"input":1.5,"output":7.5,"reasoning":7.5},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Google Gemini Flash Latest"},"~google/gemini-pro-latest":{"cost":{"cache_read":0.2,"cache_write":0.375,"input":2,"output":12,"reasoning":12,"tiers":[{"cache_read":0.4,"input":4,"output":18,"tier":{"size":200000,"type":"context"}}]},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Google Gemini Pro Latest"},"~moonshotai/kimi-latest":{"cost":{"cache_read":0.29,"input":2.5,"output":14},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"MoonshotAI Kimi Latest"},"~openai/gpt-latest":{"cost":{"cache_read":0.5,"cache_write":6.25,"input":5,"output":30,"tiers":[{"cache_read":1,"cache_write":12.5,"input":10,"output":45,"tier":{"size":272000,"type":"context"}}]},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"OpenAI GPT Latest"},"~openai/gpt-mini-latest":{"cost":{"cache_read":0.075,"input":0.75,"output":4.5},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"OpenAI GPT Mini Latest"},"~x-ai/grok-latest":{"cost":{"cache_read":0.3,"input":2,"output":6,"tiers":[{"cache_read":0.6,"input":4,"output":12,"tier":{"size":200000,"type":"context"}}]},"description":"Grok model for agentic tool use, reasoning, coding, and live assistance","name":"Grok Latest"}}},"ovhcloud":{"models":{"gpt-oss-120b":{"cost":{"input":0.09,"output":0.47},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"gpt-oss-120b"},"gpt-oss-20b":{"cost":{"input":0.05,"output":0.18},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"gpt-oss-20b"},"meta-llama-3_3-70b-instruct":{"cost":{"input":0.74,"output":0.74},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Meta-Llama-3_3-70B-Instruct"},"mistral-7b-instruct-v0.3":{"cost":{"input":0.11,"output":0.11},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral-7B-Instruct-v0.3"},"mistral-nemo-instruct-2407":{"cost":{"input":0.14,"output":0.14},"description":"Mistral model for multilingual chat, reasoning, and tool-assisted workflows","name":"Mistral-Nemo-Instruct-2407"},"mistral-small-3.2-24b-instruct-2506":{"cost":{"input":0.1,"output":0.31},"description":"Efficient Mistral model for fast chat, extraction, and production assistants","name":"Mistral-Small-3.2-24B-Instruct-2506"},"qwen2.5-vl-72b-instruct":{"cost":{"input":1.01,"output":1.01},"description":"Multimodal model for analyzing text, images, documents, and rich media","name":"Qwen2.5-VL-72B-Instruct"},"qwen3-32b":{"cost":{"input":0.09,"output":0.25},"description":"Reasoning model for deliberate analysis, multi-step problem solving, and tool use","name":"Qwen3-32B"},"qwen3-coder-30b-a3b-instruct":{"cost":{"input":0.07,"output":0.26},"description":"Coding model for repository understanding, refactors, and agentic engineering tasks","name":"Qwen3-Coder-30B-A3B-Instruct"},"qwen3.5-397b-a17b":{"cost":{"input":0.71,"output":4.25},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Qwen3.5-397B-A17B"},"qwen3.5-9b":{"cost":{"input":0.12,"output":0.18},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Qwen3.5-9B"},"qwen3.6-27b":{"cost":{"input":0.47,"output":3.19},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"Qwen3.6-27B"},"qwen3guard-gen-0.6b":{"description":"Open-weight instruction model for adaptable chat and self-hosted production workloads","name":"Qwen3Guard-Gen-0.6B"},"qwen3guard-gen-8b":{"description":"Open-weight instruction model for adaptable chat and self-hosted production workloads","name":"Qwen3Guard-Gen-8B"}}},"perplexity":{"models":{"sonar":{"cost":{"input":1.0,"output":1.0},"description":"Fast web-grounded Sonar for current answers, citations, and lightweight retrieval","name":"Sonar"},"sonar-deep-research":{"cost":{"input":2.0,"output":8.0,"reasoning":3.0},"description":"Sonar search model for current answers, retrieval, and citation-backed chat","name":"Perplexity Sonar Deep Research"},"sonar-pro":{"cost":{"input":3.0,"output":15.0},"description":"Deeper Sonar search model with broader retrieval and stronger synthesis","name":"Sonar Pro"},"sonar-reasoning-pro":{"cost":{"input":2.0,"output":8.0},"description":"Web-grounded Sonar for multi-step research questions that need cited reasoning","name":"Sonar Reasoning Pro"}}},"poe":{"models":{"anthropic/claude-haiku-3":{"cost":{"cache_read":0.021,"cache_write":0.26,"input":0.21,"output":1.1},"description":"Fast Claude model for responsive assistance, classification, and lightweight agents","name":"Claude-Haiku-3"},"anthropic/claude-haiku-3.5":{"cost":{"cache_read":0.068,"cache_write":0.85,"input":0.68,"output":3.4},"description":"Fast Claude model for responsive assistance, classification, and lightweight agents","name":"Claude-Haiku-3.5"},"anthropic/claude-haiku-4.5":{"cost":{"cache_read":0.085,"cache_write":1.1,"input":0.85,"output":4.3},"description":"Fast Claude model for responsive assistance, classification, and lightweight agents","name":"Claude-Haiku-4.5"},"anthropic/claude-opus-4":{"cost":{"cache_read":1.3,"cache_write":16,"input":13,"output":64},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude-Opus-4"},"anthropic/claude-opus-4.1":{"cost":{"cache_read":1.3,"cache_write":16,"input":13,"output":64},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude-Opus-4.1"},"anthropic/claude-opus-4.5":{"cost":{"cache_read":0.43,"cache_write":5.3,"input":4.3,"output":21},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude-Opus-4.5"},"anthropic/claude-opus-4.6":{"cost":{"cache_read":0.43,"cache_write":5.3,"input":4.3,"output":21},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude-Opus-4.6"},"anthropic/claude-opus-4.7":{"cost":{"cache_read":0.43,"cache_write":5.4,"input":4.3,"output":21},"description":"Flagship Claude model for deep reasoning, coding, and long-horizon agents","name":"Claude-Opus-4.7"},"anthropic/claude-opus-4.8":{"cost":{"input":4.2929,"output":21.4646},"description":"Top Claude Opus tier for the hardest reasoning, coding, and long-horizon agents","name":"Claude-Opus-4.8"},"anthropic/claude-sonnet-3.5":{"cost":{"cache_read":0.26,"cache_write":3.2,"input":2.6,"output":13},"description":"Legacy model retained for compatibility with older integrations","name":"Claude-Sonnet-3.5"},"anthropic/claude-sonnet-3.5-june":{"cost":{"cache_read":0.26,"cache_write":3.2,"input":2.6,"output":13},"description":"Legacy model retained for compatibility with older integrations","name":"Claude-Sonnet-3.5-June"},"anthropic/claude-sonnet-3.7":{"cost":{"cache_read":0.26,"cache_write":3.2,"input":2.6,"output":13},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude-Sonnet-3.7"},"anthropic/claude-sonnet-4":{"cost":{"cache_read":0.26,"cache_write":3.2,"input":2.6,"output":13},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude-Sonnet-4"},"anthropic/claude-sonnet-4.5":{"cost":{"cache_read":0.26,"cache_write":3.2,"input":2.6,"output":13},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude-Sonnet-4.5"},"anthropic/claude-sonnet-4.6":{"cost":{"cache_read":0.26,"cache_write":3.2,"input":2.6,"output":13},"description":"Balanced Claude model for coding, analysis, agent workflows, and cost control","name":"Claude-Sonnet-4.6"},"cerebras/gpt-oss-120b-cs":{"cost":{"input":0.35,"output":0.75},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"GPT-OSS-120B-CS"},"cerebras/llama-3.1-8b-cs":{"cost":{"input":0.1,"output":0.1},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama-3.1-8B-CS"},"cerebras/llama-3.3-70b-cs":{"description":"Legacy model retained for compatibility with older integrations","name":"llama-3.3-70b-cs"},"cerebras/qwen3-235b-2507-cs":{"description":"Legacy model retained for compatibility with older integrations","name":"qwen3-235b-2507-cs"},"cerebras/qwen3-32b-cs":{"description":"Legacy model retained for compatibility with older integrations","name":"qwen3-32b-cs"},"elevenlabs/elevenlabs-music":{"description":"Speech generation model for controllable voice, narration, and audio delivery","name":"ElevenLabs-Music"},"elevenlabs/elevenlabs-v2.5-turbo":{"description":"Speech generation model for controllable voice, narration, and audio delivery","name":"ElevenLabs-v2.5-Turbo"},"elevenlabs/elevenlabs-v3":{"description":"Speech generation model for controllable voice, narration, and audio delivery","name":"ElevenLabs-v3"},"empiriolabs/deepseek-v4-flash-el":{"cost":{"input":0.14,"output":0.28},"description":"Fast DeepSeek model for efficient chat, coding help, and agent loops","name":"DeepSeek-V4-Flash-EL"},"empiriolabs/deepseek-v4-pro-el":{"cost":{"input":1.67,"output":3.33},"description":"Flagship DeepSeek model for coding, reasoning, and agentic work","name":"DeepSeek-V4-Pro-EL"},"fireworks-ai/kimi-k2.5-fw":{"cost":{"input":0,"output":0},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi-K2.5-FW"},"google/gemini-2.0-flash":{"cost":{"input":0.1,"output":0.42},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini-2.0-Flash"},"google/gemini-2.0-flash-lite":{"cost":{"input":0.052,"output":0.21},"description":"Low-latency Gemini model for high-volume multimodal and agent workloads","name":"Gemini-2.0-Flash-Lite"},"google/gemini-2.5-flash":{"cost":{"cache_read":0.021,"input":0.21,"output":1.8},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini-2.5-Flash"},"google/gemini-2.5-flash-lite":{"cost":{"input":0.07,"output":0.28},"description":"Low-latency Gemini model for high-volume multimodal and agent workloads","name":"Gemini-2.5-Flash-Lite"},"google/gemini-2.5-pro":{"cost":{"cache_read":0.087,"input":0.87,"output":7},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Gemini-2.5-Pro"},"google/gemini-3-flash":{"cost":{"cache_read":0.04,"input":0.4,"output":2.4},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini-3-Flash"},"google/gemini-3-pro":{"cost":{"cache_read":0.16,"input":1.6,"output":9.6},"description":"Legacy model retained for compatibility with older integrations","name":"Gemini-3-Pro"},"google/gemini-3.1-flash-lite":{"cost":{"input":0.25,"output":1.5},"description":"Low-latency Gemini model for high-volume multimodal and agent workloads","name":"Gemini-3.1-Flash-Lite"},"google/gemini-3.1-pro":{"cost":{"cache_read":0.2,"input":2,"output":12},"description":"Advanced Gemini model for complex reasoning, coding, and multimodal analysis","name":"Gemini-3.1-Pro"},"google/gemini-3.5-flash":{"cost":{"cache_read":0.1515,"input":1.5152,"output":9.0909},"description":"Fast Gemini model balancing multimodal reasoning, tool use, and cost","name":"Gemini-3.5-Flash"},"google/gemini-deep-research":{"cost":{"input":1.6,"output":9.6},"description":"Legacy model retained for compatibility with older integrations","name":"gemini-deep-research"},"google/gemma-4-31b":{"cost":{"input":0,"output":0},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma-4-31B"},"google/imagen-3":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Imagen-3"},"google/imagen-3-fast":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Imagen-3-Fast"},"google/imagen-4":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Imagen-4"},"google/imagen-4-fast":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Imagen-4-Fast"},"google/imagen-4-ultra":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Imagen-4-Ultra"},"google/lyria":{"description":"Speech generation model for controllable voice, narration, and audio delivery","name":"Lyria"},"google/nano-banana":{"cost":{"cache_read":0.021,"input":0.21,"output":1.8},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Nano-Banana"},"google/nano-banana-pro":{"cost":{"cache_read":0.2,"input":2,"output":12},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Nano-Banana-Pro"},"google/veo-2":{"description":"Video model for prompt-guided generation, editing, and motion workflows","name":"Veo-2"},"google/veo-3":{"description":"Video model for prompt-guided generation, editing, and motion workflows","name":"Veo-3"},"google/veo-3-fast":{"description":"Video model for prompt-guided generation, editing, and motion workflows","name":"Veo-3-Fast"},"google/veo-3.1":{"description":"Video model for prompt-guided generation, editing, and motion workflows","name":"Veo-3.1"},"google/veo-3.1-fast":{"description":"Video model for prompt-guided generation, editing, and motion workflows","name":"Veo-3.1-Fast"},"ideogramai/ideogram":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Ideogram"},"ideogramai/ideogram-v2":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Ideogram-v2"},"ideogramai/ideogram-v2a":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Ideogram-v2a"},"ideogramai/ideogram-v2a-turbo":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Ideogram-v2a-Turbo"},"lumalabs/ray2":{"description":"Video model for prompt-guided generation, editing, and motion workflows","name":"Ray2"},"novita/deepseek-v3.2":{"cost":{"cache_read":0.13,"input":0.27,"output":0.4},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"DeepSeek-V3.2"},"novita/glm-4.6":{"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM-4.6"},"novita/glm-4.6v":{"description":"GLM vision model for visual reasoning, documents, and multimodal agents","name":"glm-4.6v"},"novita/glm-4.7":{"description":"Legacy model retained for compatibility with older integrations","name":"glm-4.7"},"novita/glm-4.7-flash":{"description":"Efficient GLM model for fast reasoning, coding, and agent workflows","name":"glm-4.7-flash"},"novita/glm-4.7-n":{"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"glm-4.7-n"},"novita/glm-5":{"cost":{"cache_read":0.2,"input":1,"output":3.2},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"GLM-5"},"novita/kimi-k2-thinking":{"description":"Kimi reasoning model for long-horizon research, planning, and tool use","name":"kimi-k2-thinking"},"novita/kimi-k2.5":{"cost":{"cache_read":0.1,"input":0.6,"output":3},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi-K2.5"},"novita/kimi-k2.6":{"cost":{"cache_read":0.16,"input":0.96,"output":4.04},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"Kimi-K2.6"},"novita/minimax-m2.1":{"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"minimax-m2.1"},"openai/chatgpt-4o-latest":{"cost":{"input":4.5,"output":14},"description":"Legacy model retained for compatibility with older integrations","name":"ChatGPT-4o-Latest"},"openai/dall-e-3":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"DALL-E-3"},"openai/gpt-3.5-turbo":{"cost":{"input":0.45,"output":1.4},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-3.5-Turbo"},"openai/gpt-3.5-turbo-instruct":{"cost":{"input":1.4,"output":1.8},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-3.5-Turbo-Instruct"},"openai/gpt-3.5-turbo-raw":{"cost":{"input":0.45,"output":1.4},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-3.5-Turbo-Raw"},"openai/gpt-4-classic":{"cost":{"input":27,"output":54},"description":"Legacy model retained for compatibility with older integrations","name":"GPT-4-Classic"},"openai/gpt-4-classic-0314":{"cost":{"input":27,"output":54},"description":"Legacy model retained for compatibility with older integrations","name":"GPT-4-Classic-0314"},"openai/gpt-4-turbo":{"cost":{"input":9,"output":27},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-4-Turbo"},"openai/gpt-4.1":{"cost":{"cache_read":0.45,"input":1.8,"output":7.2},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-4.1"},"openai/gpt-4.1-mini":{"cost":{"cache_read":0.09,"input":0.36,"output":1.4},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-4.1-mini"},"openai/gpt-4.1-nano":{"cost":{"cache_read":0.022,"input":0.09,"output":0.36},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-4.1-nano"},"openai/gpt-4o":{"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-4o"},"openai/gpt-4o-aug":{"cost":{"cache_read":1.1,"input":2.2,"output":9},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-4o-Aug"},"openai/gpt-4o-mini":{"cost":{"cache_read":0.068,"input":0.14,"output":0.54},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-4o-mini"},"openai/gpt-4o-mini-search":{"cost":{"input":0.14,"output":0.54},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-4o-mini-Search"},"openai/gpt-4o-search":{"cost":{"input":2.2,"output":9},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-4o-Search"},"openai/gpt-5":{"cost":{"cache_read":0.11,"input":1.1,"output":9},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-5"},"openai/gpt-5-chat":{"cost":{"cache_read":0.11,"input":1.1,"output":9},"description":"Chat-tuned GPT model for conversational assistance, writing, and tool workflows","name":"GPT-5-Chat"},"openai/gpt-5-codex":{"cost":{"input":1.1,"output":9},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5-Codex"},"openai/gpt-5-mini":{"cost":{"cache_read":0.022,"input":0.22,"output":1.8},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-5-mini"},"openai/gpt-5-nano":{"cost":{"cache_read":0.0045,"input":0.045,"output":0.36},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-5-nano"},"openai/gpt-5-pro":{"cost":{"input":14,"output":110},"description":"Frontier GPT model for professional reasoning, coding, and multimodal work","name":"GPT-5-Pro"},"openai/gpt-5.1":{"cost":{"cache_read":0.11,"input":1.1,"output":9},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-5.1"},"openai/gpt-5.1-codex":{"cost":{"cache_read":0.11,"input":1.1,"output":9},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.1-Codex"},"openai/gpt-5.1-codex-max":{"cost":{"cache_read":0.11,"input":1.1,"output":9},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.1-Codex-Max"},"openai/gpt-5.1-codex-mini":{"cost":{"cache_read":0.022,"input":0.22,"output":1.8},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.1-Codex-Mini"},"openai/gpt-5.1-instant":{"cost":{"cache_read":0.11,"input":1.1,"output":9},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-5.1-Instant"},"openai/gpt-5.2":{"cost":{"cache_read":0.16,"input":1.6,"output":13},"description":"GPT model for general reasoning, writing, coding, and tool-assisted tasks","name":"GPT-5.2"},"openai/gpt-5.2-codex":{"cost":{"cache_read":0.16,"input":1.6,"output":13},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.2-Codex"},"openai/gpt-5.2-instant":{"cost":{"cache_read":0.16,"input":1.6,"output":13},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-5.2-Instant"},"openai/gpt-5.2-pro":{"cost":{"input":19,"output":150},"description":"Frontier GPT model for professional reasoning, coding, and multimodal work","name":"GPT-5.2-Pro"},"openai/gpt-5.3-codex":{"cost":{"cache_read":0.16,"input":1.6,"output":13},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.3-Codex"},"openai/gpt-5.3-codex-spark":{"cost":{"input":0,"output":0},"description":"Coding-optimized GPT model for repository edits, reviews, and agentic software work","name":"GPT-5.3-Codex-Spark"},"openai/gpt-5.3-instant":{"cost":{"cache_read":0.16,"input":1.6,"output":13},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-5.3-Instant"},"openai/gpt-5.4":{"cost":{"cache_read":0.22,"input":2.2,"output":14},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"GPT-5.4"},"openai/gpt-5.4-mini":{"cost":{"cache_read":0.068,"input":0.68,"output":4},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-5.4-Mini"},"openai/gpt-5.4-nano":{"cost":{"cache_read":0.018,"input":0.18,"output":1.1},"description":"Compact GPT model for low-latency assistance and high-volume workloads","name":"GPT-5.4-Nano"},"openai/gpt-5.4-pro":{"cost":{"input":27,"output":160},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"GPT-5.4-Pro"},"openai/gpt-5.5":{"cost":{"cache_read":0.4545,"input":4.5455,"output":27.2727},"description":"Default frontier GPT for coding, computer use, research, and knowledge work","name":"GPT-5.5"},"openai/gpt-5.5-pro":{"cost":{"input":27.2727,"output":163.6364},"description":"Highest-accuracy GPT-5.5 tier for slower, precision-heavy reasoning and coding","name":"GPT-5.5-Pro"},"openai/gpt-image-1":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"GPT-Image-1"},"openai/gpt-image-1-mini":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"GPT-Image-1-Mini"},"openai/gpt-image-1.5":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"gpt-image-1.5"},"openai/gpt-image-2":{"cost":{"cache_read":1.2626,"input":5.0505,"output":32.3232},"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"GPT-Image-2"},"openai/o1":{"cost":{"input":14,"output":54},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"o1"},"openai/o1-pro":{"cost":{"input":140,"output":540},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"o1-pro"},"openai/o3":{"cost":{"cache_read":0.45,"input":1.8,"output":7.2},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"o3"},"openai/o3-deep-research":{"cost":{"cache_read":2.2,"input":9,"output":36},"description":"Research model for long-horizon investigation, synthesis, and analytical reports","name":"o3-deep-research"},"openai/o3-mini":{"cost":{"input":0.99,"output":4},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"o3-mini"},"openai/o3-mini-high":{"cost":{"input":0.99,"output":4},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"o3-mini-high"},"openai/o3-pro":{"cost":{"input":18,"output":72},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"o3-pro"},"openai/o4-mini":{"cost":{"cache_read":0.25,"input":0.99,"output":4},"description":"O-series reasoning model for hard analysis, math, coding, and planning","name":"o4-mini"},"openai/o4-mini-deep-research":{"cost":{"cache_read":0.45,"input":1.8,"output":7.2},"description":"Research model for long-horizon investigation, synthesis, and analytical reports","name":"o4-mini-deep-research"},"openai/sora-2":{"description":"Video model for prompt-guided generation, editing, and motion workflows","name":"Sora-2"},"openai/sora-2-pro":{"description":"Video model for prompt-guided generation, editing, and motion workflows","name":"Sora-2-Pro"},"poetools/claude-code":{"description":"Claude model for careful reasoning, writing, coding, and tool use","name":"claude-code"},"runwayml/runway":{"description":"Video model for prompt-guided generation, editing, and motion workflows","name":"Runway"},"runwayml/runway-gen-4-turbo":{"description":"Video model for prompt-guided generation, editing, and motion workflows","name":"Runway-Gen-4-Turbo"},"stabilityai/stablediffusionxl":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"StableDiffusionXL"},"topazlabs-co/topazlabs":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"TopazLabs"},"trytako/tako":{"description":"Tool-capable chat model for instruction following and agentic application workflows","name":"Tako"},"xai/grok-3":{"cost":{"cache_read":0.75,"input":3,"output":15},"description":"Grok model for agentic tool use, reasoning, coding, and live assistance","name":"Grok 3"},"xai/grok-3-mini":{"cost":{"cache_read":0.075,"input":0.3,"output":0.5},"description":"Fast Grok model for responsive chat, reasoning, and tool-assisted work","name":"Grok 3 Mini"},"xai/grok-4":{"cost":{"cache_read":0.75,"input":3,"output":15},"description":"Grok model for agentic tool use, reasoning, coding, and live assistance","name":"Grok-4"},"xai/grok-4-fast-non-reasoning":{"cost":{"cache_read":0.05,"input":0.2,"output":0.5},"description":"Fast Grok model for responsive chat, reasoning, and tool-assisted work","name":"Grok-4-Fast-Non-Reasoning"},"xai/grok-4-fast-reasoning":{"cost":{"cache_read":0.05,"input":0.2,"output":0.5},"description":"Fast Grok model for responsive chat, reasoning, and tool-assisted work","name":"Grok-4-Fast-Reasoning"},"xai/grok-4.1-fast-non-reasoning":{"description":"Fast Grok model for responsive chat, reasoning, and tool-assisted work","name":"Grok-4.1-Fast-Non-Reasoning"},"xai/grok-4.1-fast-reasoning":{"description":"Fast Grok model for responsive chat, reasoning, and tool-assisted work","name":"Grok-4.1-Fast-Reasoning"},"xai/grok-4.20-multi-agent":{"cost":{"cache_read":0.2,"input":2,"output":6},"description":"Grok model for agentic tool use, reasoning, coding, and live assistance","name":"Grok-4.20-Multi-Agent"},"xai/grok-code-fast-1":{"cost":{"cache_read":0.02,"input":0.2,"output":1.5},"description":"Fast Grok model for responsive chat, reasoning, and tool-assisted work","name":"Grok Code Fast 1"}}},"scaleway":{"models":{"bge-multilingual-gemma2":{"cost":{"input":0.1,"output":0.0},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"BGE Multilingual Gemma2"},"gemma-4-26b-a4b-it":{"cost":{"input":0.25,"output":0.5},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 4 26B A4B IT"},"glm-5.2":{"cost":{"input":1.8,"output":5.5},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM-5.2"},"gpt-oss-120b":{"cost":{"input":0.15,"output":0.6},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"GPT-OSS 120B"},"llama-3.3-70b-instruct":{"cost":{"input":0.9,"output":0.9},"description":"Open Llama instruction model for multilingual chat, reasoning, and coding","name":"Llama-3.3-70B-Instruct"},"mistral-medium-3.5-128b":{"cost":{"input":1.5,"output":7.5},"description":"Balanced Mistral model for enterprise assistants, multilingual work, and tools","name":"Mistral Medium 3.5 128B"},"mistral-small-3.2-24b-instruct-2506":{"cost":{"input":0.15,"output":0.35},"description":"Efficient Mistral model for fast chat, extraction, and production assistants","name":"Mistral Small 3.2 24B Instruct (2506)"},"pixtral-12b-2409":{"cost":{"input":0.2,"output":0.2},"description":"Mistral vision-language model for image understanding and multimodal chat","name":"Pixtral 12B 2409"},"qwen3-235b-a22b-instruct-2507":{"cost":{"input":0.75,"output":2.25,"reasoning":8.4},"description":"Large open Qwen MoE for multilingual reasoning, coding, and tool use","name":"Qwen3 235B A22B Instruct 2507"},"qwen3-coder-30b-a3b-instruct":{"cost":{"input":0.2,"output":0.8},"description":"Smaller Qwen coder for efficient local agents and repo-level fixes","name":"Qwen3-Coder 30B-A3B Instruct"},"qwen3-embedding-8b":{"cost":{"input":0.1,"output":0.0},"description":"Embedding model for semantic search, retrieval, clustering, and ranking pipelines","name":"Qwen3 Embedding 8B"},"qwen3.5-397b-a17b":{"cost":{"input":0.6,"output":3.6},"description":"Large open Qwen multimodal MoE for visual agents and long technical tasks","name":"Qwen3.5 397B A17B"},"qwen3.6-35b-a3b":{"cost":{"input":0.25,"output":1.5},"description":"Open multimodal Qwen MoE for local agents that need vision, audio, and code","name":"Qwen3.6 35B A3B"},"whisper-large-v3":{"cost":{"input":0.003,"output":0.0},"description":"Speech transcription model for accurate audio-to-text and captioning workflows","name":"Whisper Large v3"}}},"siliconflow":{"models":{"ByteDance-Seed/Seed-OSS-36B-Instruct":{"cost":{"input":0.21,"output":0.57},"description":"Tool-capable chat model for instruction following and agentic application workflows","name":"ByteDance-Seed/Seed-OSS-36B-Instruct"},"MiniMaxAI/MiniMax-M2.5":{"cost":{"cache_read":0.03,"input":0.3,"output":1.2},"description":"MiniMax model for chat, coding, office work, and agentic tasks","name":"MiniMaxAI/MiniMax-M2.5"},"Qwen/Qwen2.5-72B-Instruct":{"cost":{"input":0.59,"output":0.59},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen/Qwen2.5-72B-Instruct"},"Qwen/Qwen2.5-7B-Instruct":{"cost":{"input":0.05,"output":0.05},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen/Qwen2.5-7B-Instruct"},"Qwen/Qwen3-14B":{"cost":{"input":0.07,"output":0.28},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen/Qwen3-14B"},"Qwen/Qwen3-235B-A22B-Thinking-2507":{"cost":{"input":0.13,"output":0.6},"description":"Qwen reasoning model for deliberate problem solving, math, and coding","name":"Qwen/Qwen3-235B-A22B-Thinking-2507"},"Qwen/Qwen3-30B-A3B-Instruct-2507":{"cost":{"input":0.09,"output":0.3},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen/Qwen3-30B-A3B-Instruct-2507"},"Qwen/Qwen3-32B":{"cost":{"input":0.14,"output":0.57},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen/Qwen3-32B"},"Qwen/Qwen3-8B":{"cost":{"input":0.06,"output":0.06},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen/Qwen3-8B"},"Qwen/Qwen3-Coder-30B-A3B-Instruct":{"cost":{"input":0.07,"output":0.28},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen/Qwen3-Coder-30B-A3B-Instruct"},"Qwen/Qwen3-Coder-480B-A35B-Instruct":{"cost":{"input":0.25,"output":1.0},"description":"Qwen coding model for software agents, repository edits, and code reasoning","name":"Qwen/Qwen3-Coder-480B-A35B-Instruct"},"Qwen/Qwen3-VL-235B-A22B-Instruct":{"cost":{"input":0.3,"output":1.5},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen/Qwen3-VL-235B-A22B-Instruct"},"Qwen/Qwen3-VL-235B-A22B-Thinking":{"cost":{"input":0.45,"output":3.5},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen/Qwen3-VL-235B-A22B-Thinking"},"Qwen/Qwen3-VL-30B-A3B-Instruct":{"cost":{"input":0.29,"output":1.0},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen/Qwen3-VL-30B-A3B-Instruct"},"Qwen/Qwen3-VL-30B-A3B-Thinking":{"cost":{"input":0.29,"output":1.0},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen/Qwen3-VL-30B-A3B-Thinking"},"Qwen/Qwen3-VL-32B-Instruct":{"cost":{"input":0.2,"output":0.6},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen/Qwen3-VL-32B-Instruct"},"Qwen/Qwen3-VL-32B-Thinking":{"cost":{"input":0.2,"output":1.5},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen/Qwen3-VL-32B-Thinking"},"Qwen/Qwen3-VL-8B-Instruct":{"cost":{"input":0.18,"output":0.68},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen/Qwen3-VL-8B-Instruct"},"Qwen/Qwen3.5-122B-A10B":{"cost":{"input":0.26,"output":2.08},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 122B-A10B"},"Qwen/Qwen3.5-27B":{"cost":{"input":0.25,"output":2.0},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 27B"},"Qwen/Qwen3.5-35B-A3B":{"cost":{"input":0.24,"output":1.8},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.5 35B-A3B"},"Qwen/Qwen3.5-397B-A17B":{"cost":{"input":0.39,"output":2.34},"description":"Large open Qwen multimodal MoE for visual agents and long technical tasks","name":"Qwen3.5 397B-A17B"},"Qwen/Qwen3.5-9B":{"cost":{"input":0.1,"output":0.15},"description":"Qwen instruction model for multilingual chat, reasoning, and tool use","name":"Qwen/Qwen3.5-9B"},"Qwen/Qwen3.6-27B":{"cost":{"input":0.3,"output":3.2},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.6 27B"},"Qwen/Qwen3.6-35B-A3B":{"cost":{"input":0.2,"output":1.6},"description":"Open multimodal Qwen MoE for local agents that need vision, audio, and code","name":"Qwen3.6 35B-A3B"},"baidu/ERNIE-4.5-300B-A47B":{"cost":{"input":0.28,"output":1.1},"description":"Tool-capable chat model for instruction following and agentic application workflows","name":"baidu/ERNIE-4.5-300B-A47B"},"deepseek-ai/DeepSeek-R1":{"cost":{"input":0.5,"output":2.18},"description":"DeepSeek reasoning model for multi-step analysis, math, coding, and tools","name":"deepseek-ai/DeepSeek-R1"},"deepseek-ai/DeepSeek-V3":{"cost":{"input":0.25,"output":1.0},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"deepseek-ai/DeepSeek-V3"},"deepseek-ai/DeepSeek-V3.1":{"cost":{"input":0.27,"output":1.0},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"deepseek-ai/DeepSeek-V3.1"},"deepseek-ai/DeepSeek-V3.1-Terminus":{"cost":{"input":0.27,"output":1.0},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"deepseek-ai/DeepSeek-V3.1-Terminus"},"deepseek-ai/DeepSeek-V3.2":{"cost":{"input":0.27,"output":0.42},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"deepseek-ai/DeepSeek-V3.2"},"deepseek-ai/DeepSeek-V3.2-Exp":{"cost":{"input":0.27,"output":0.41},"description":"DeepSeek chat model for instruction following, coding, and analysis","name":"deepseek-ai/DeepSeek-V3.2-Exp"},"deepseek-ai/DeepSeek-V4-Flash":{"cost":{"cache_read":0.028,"input":0.14,"output":0.28},"description":"Fast DeepSeek V4 lane for economical reasoning, coding, and long-context work","name":"DeepSeek V4 Flash"},"deepseek-ai/DeepSeek-V4-Pro":{"cost":{"cache_read":0.145,"input":1.74,"output":3.48},"description":"Open MoE flagship with million-token context for coding and long agent runs","name":"DeepSeek V4 Pro"},"google/gemma-4-26B-A4B-it":{"cost":{"input":0.12,"output":0.4},"description":"Open Gemma instruction model for efficient chat and self-hosted deployments","name":"Gemma 4 26B A4B IT"},"google/gemma-4-31B-it":{"cost":{"input":0.13,"output":0.4},"description":"Largest Gemma 4 instruction model for open, self-hosted chat and reasoning","name":"Gemma 4 31B IT"},"inclusionAI/Ling-flash-2.0":{"cost":{"input":0.14,"output":0.57},"description":"Efficient model for low-latency assistance, extraction, and routine automation","name":"inclusionAI/Ling-flash-2.0"},"moonshotai/Kimi-K2.5":{"cost":{"cache_read":0.07,"input":0.45,"output":2.25},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"moonshotai/Kimi-K2.5"},"moonshotai/Kimi-K2.6":{"cost":{"cache_read":0.2,"input":0.77,"output":4.0},"description":"Kimi multimodal agent model for visual understanding, coding, and planning","name":"moonshotai/Kimi-K2.6"},"openai/gpt-oss-120b":{"cost":{"input":0.05,"output":0.45},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"openai/gpt-oss-120b"},"openai/gpt-oss-20b":{"cost":{"input":0.04,"output":0.18},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"openai/gpt-oss-20b"},"stepfun-ai/Step-3.5-Flash":{"cost":{"input":0.1,"output":0.3},"description":"StepFun flash model for efficient multimodal reasoning, coding, and tool use","name":"stepfun-ai/Step-3.5-Flash"},"tencent/Hunyuan-A13B-Instruct":{"cost":{"input":0.14,"output":0.57},"description":"Tencent Hy reasoning model for coding, instruction following, and agent tasks","name":"tencent/Hunyuan-A13B-Instruct"},"tencent/Hy3-preview":{"cost":{"cache_read":0.029,"input":0.066,"output":0.26},"description":"Tencent Hy reasoning model for coding, instruction following, and agent tasks","name":"Hy3 preview"},"zai-org/GLM-4.5-Air":{"cost":{"input":0.14,"output":0.86},"description":"Efficient GLM model for fast reasoning, coding, and agent workflows","name":"zai-org/GLM-4.5-Air"},"zai-org/GLM-5":{"cost":{"cache_read":0.2,"input":0.95,"output":2.55},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"zai-org/GLM-5"},"zai-org/GLM-5.1":{"cost":{"cache_read":0.26,"cache_write":0,"input":1.4,"output":4.4},"description":"Flagship GLM model for hybrid reasoning, coding, and agentic engineering","name":"zai-org/GLM-5.1"},"zai-org/GLM-5.2":{"cost":{"cache_read":0.26,"cache_write":0,"input":1.4,"output":4.4},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM-5.2"},"zai-org/GLM-5V-Turbo":{"cost":{"cache_read":0.24,"cache_write":0,"input":1.2,"output":4},"description":"GLM vision model for visual reasoning, documents, and multimodal agents","name":"zai-org/GLM-5V-Turbo"}}},"synthetic":{"models":{"hf:MiniMaxAI/MiniMax-M3":{"cost":{"cache_read":0.6,"input":0.6,"output":1.2},"description":"MiniMax multimodal model for long-context coding, perception, and agent planning","name":"MiniMax-M3"},"hf:Qwen/Qwen3.6-27B":{"cost":{"cache_read":0.45,"input":0.45,"output":3.6},"description":"Qwen vision-language model for visual reasoning, documents, and agent tasks","name":"Qwen3.6 27B"},"hf:moonshotai/Kimi-K2.7-Code":{"cost":{"cache_read":0.95,"input":0.95,"output":4},"description":"Coding-focused Kimi model, stronger on long-horizon repo work with less overthinking","name":"Kimi K2.7 Code"},"hf:moonshotai/Kimi-K3":{"cost":{"cache_read":0.45,"input":3,"output":15},"description":"Multimodal Kimi model with 1M context and toggleable max-effort thinking for long-horizon agent work","name":"Kimi K3"},"hf:nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4":{"cost":{"cache_read":0.3,"input":0.3,"output":1},"description":"Nemotron middle tier for collaborative agents and high-volume reasoning workloads","name":"Nemotron 3 Super 120B A12B"},"hf:openai/gpt-oss-120b":{"cost":{"cache_read":0.1,"input":0.1,"output":0.1},"description":"Open-weight GPT model for self-hosted reasoning and instruction-following workloads","name":"GPT OSS 120B"},"hf:zai-org/GLM-4.7-Flash":{"cost":{"cache_read":0.1,"input":0.1,"output":0.5},"description":"Budget GLM lane for fast coding help, routing, and everyday automation","name":"GLM-4.7-Flash"},"hf:zai-org/GLM-5.2":{"cost":{"cache_read":1.4,"input":1.4,"output":4.4},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM-5.2"}}},"v0":{"models":{"v0-1.0-md":{"cost":{"input":3.0,"output":15.0},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"v0-1.0-md"},"v0-1.5-lg":{"cost":{"input":15.0,"output":75.0},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"v0-1.5-lg"},"v0-1.5-md":{"cost":{"input":3.0,"output":15.0},"description":"Multimodal reasoning model for visual analysis, planning, and tool use","name":"v0-1.5-md"}}},"wandb":{"models":{"JetBrains/Mellum2-12B-A2.5B-Instruct":{"cost":{"cache_read":0.05,"input":0.05,"output":0.1},"description":"Mellum2-12B-A2.5B-Instruct is a fast MoE model with 131K context built for coding, tool use, and low-latency AI workflows.","name":"Mellum2 12B A2.5B"},"MiniMaxAI/MiniMax-M2.5":{"cost":{"cache_read":0.3,"input":0.3,"output":1.2},"description":"MoE model with a highly sparse architecture designed for high-throughput and low latency with strong coding capabilities.","name":"MiniMax M2.5"},"MiniMaxAI/MiniMax-M3":{"cost":{"cache_read":0.05,"input":0.23,"output":0.96},"description":"MiniMax M3 is a multimodal MoE model with 23B active parameters optimized for coding and agentic workflows.","name":"MiniMax M3"},"OpenPipe/Qwen3-14B-Instruct":{"cost":{"cache_read":0.05,"input":0.05,"output":0.22},"description":"An efficient multilingual, dense, instruction-tuned model, optimized by OpenPipe for building agents with finetuning.","name":"Qwen3 14B Instruct"},"Qwen/Qwen3-30B-A3B-Instruct-2507":{"cost":{"cache_read":0.1,"input":0.1,"output":0.3},"description":"Qwen3-30B-A3B-Instruct-2507 is a 30.5B MoE instruction-tuned model with enhanced reasoning, coding, and long-context understanding.","name":"Qwen3 30B A3B Instruct 2507"},"Qwen/Qwen3-Coder-480B-A35B-Instruct":{"cost":{"cache_read":1,"input":1,"output":1.5},"description":"Mixture-of-Experts model optimized for agentic coding tasks such as function calling, tool use, and long-context reasoning.","name":"Qwen3 Coder 480B A35B"},"Qwen/Qwen3.5-35B-A3B":{"cost":{"cache_read":0.25,"input":0.25,"output":1.25},"description":"Qwen3.5-35B-A3B is an open-weights multimodal MoE model built for efficient, high-throughput inference across chat, reasoning, and agentic tasks.","name":"Qwen3.5-35B-A3B"},"Qwen/Qwen3.6-27B":{"cost":{"cache_read":0.12,"input":0.6,"output":3.6},"description":"Qwen3.6-27B is a 27B dense multimodal model with 262K context built for flagship-level agentic coding.","name":"Qwen3.6 27B"},"Qwen/Qwen3.6-35B-A3B":{"cost":{"cache_read":0.25,"input":0.25,"output":1.25},"description":"Qwen3.6-35B-A3B is an MoE multimodal model with 262K context optimized for agentic coding workflows.","name":"Qwen3.6 35B A3B"},"deepseek-ai/DeepSeek-V3.1":{"cost":{"cache_read":0.55,"input":0.55,"output":1.65},"description":"A large hybrid model that supports both thinking and non-thinking modes via prompt templates.","name":"DeepSeek V3.1"},"deepseek-ai/DeepSeek-V4-Flash":{"cost":{"cache_read":0.07,"input":0.14,"output":0.28},"description":"DeepSeek V4-Flash is an MoE model with 1M context length great for coding, reasoning, and agentic workloads.","name":"DeepSeek V4 Flash"},"deepseek-ai/DeepSeek-V4-Flash-0731":{"cost":{"cache_read":0.07,"input":0.13,"output":0.28},"description":"DeepSeek V4-Flash-0731 is an MoE model great for coding, reasoning, and agentic workloads.","name":"DeepSeek V4 Flash 0731"},"deepseek-ai/DeepSeek-V4-Pro":{"cost":{"cache_read":0.2,"input":1.15,"output":2.55},"description":"DeepSeek V4-Pro is a 1.6T-parameter MoE model with 49B active parameters excelling at advanced reasoning, coding, and complex agentic workloads.","name":"DeepSeek V4 Pro"},"google/gemma-4-31B-it":{"cost":{"cache_read":0.1,"input":0.1,"output":0.34},"description":"Gemma 4 31B Dense is designed for advanced reasoning, agentic workflows, and longer context and is natively trained on 140+ languages.","name":"Gemma 4 31B"},"ibm-granite/granite-4.1-8b":{"cost":{"cache_read":0.05,"input":0.05,"output":0.1},"description":"Granite 4.1 8B is a long-context instruct model capable of enhanced tool calling, instruction following, and chat capabilities.","name":"Granite 4.1 8B"},"meta-llama/Llama-3.1-70B-Instruct":{"cost":{"cache_read":0.8,"input":0.8,"output":0.8},"description":"Efficient conversational model optimized for responsive multilingual chatbot interactions.","name":"Llama 3.1 70B"},"meta-llama/Llama-3.1-8B-Instruct":{"cost":{"cache_read":0.22,"input":0.22,"output":0.22},"description":"Efficient conversational model optimized for responsive multilingual chatbot interactions.","name":"Llama 3.1 8B"},"meta-llama/Llama-3.3-70B-Instruct":{"cost":{"cache_read":0.71,"input":0.71,"output":0.71},"description":"Multilingual model excelling in conversational tasks, detailed instruction-following, and coding.","name":"Llama 3.3 70B"},"moonshotai/Kimi-K2.6":{"cost":{"cache_read":0.15,"input":0.65,"output":3.41},"description":"Kimi K2.6 is a multimodal Mixture-of-Experts language model featuring 32 billion activated parameters and a total of 1 trillion parameters.","name":"Kimi K2.6"},"moonshotai/Kimi-K2.7-Code":{"cost":{"cache_read":0.15,"input":0.71,"output":3.5},"description":"Kimi K2.7 Code is a 1T-parameter MoE model with 32B active parameters purpose-built for long-horizon agentic coding and software engineering.","name":"Kimi K2.7 Code"},"moonshotai/Kimi-K3":{"cost":{"cache_read":0.3,"input":3,"output":15},"description":"Kimi K3 is a 2.8T-parameter multimodal MoE model with 104B active parameters built for long-horizon coding and agentic workflows.","name":"Kimi K3"},"nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8":{"cost":{"cache_read":0.2,"input":0.2,"output":0.8},"description":"Nemotron 3 is a LatentMoE model designed to deliver strong agentic, reasoning, and conversational capabilities.","name":"Nemotron 3 Super"},"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B":{"cost":{"cache_read":0.15,"input":0.75,"output":2.75},"description":"Nemotron 3 Ultra is a powerful MoE model designed for long-running agents across coding, deep research, and enterprise automation.","name":"Nemotron 3 Ultra"},"openai/gpt-oss-120b":{"cost":{"cache_read":0.03,"input":0.03,"output":0.17},"description":"Efficient Mixture-of-Experts model designed for high-reasoning, agentic and general-purpose use cases.","name":"gpt-oss-120b"},"openai/gpt-oss-20b":{"cost":{"cache_read":0.03,"input":0.03,"output":0.13},"description":"Lower latency Mixture-of-Experts model trained on OpenAI's Harmony response format with reasoning capabilities.","name":"gpt-oss-20b"},"zai-org/GLM-5.1":{"cost":{"cache_read":0.26,"input":1.4,"output":4.4},"description":"Powerful MoE model for long-horizon agentic engineering and advanced reasoning.","name":"GLM 5.1"},"zai-org/GLM-5.2":{"cost":{"cache_read":0.14,"input":0.76,"output":2.42},"description":"GLM-5.2 is a Mixture-of-Experts language model featuring 40 billion activated parameters and a total of 744 billion parameters.","name":"GLM 5.2"}}},"xai":{"models":{"grok-4.20-0309-non-reasoning":{"cost":{"cache_read":0.2,"input":1.25,"output":2.5,"tiers":[{"cache_read":0.4,"input":2.5,"output":5,"tier":{"size":200000,"type":"context"}}]},"description":"Grok model for agentic tool use, reasoning, coding, and live assistance","name":"Grok 4.20 (Non-Reasoning)"},"grok-4.20-0309-reasoning":{"cost":{"cache_read":0.2,"input":1.25,"output":2.5,"tiers":[{"cache_read":0.4,"input":2.5,"output":5,"tier":{"size":200000}}]},"description":"Reasoning Grok for document-heavy analysis and long-horizon tool use","name":"Grok 4.20 (Reasoning)"},"grok-4.20-multi-agent-0309":{"cost":{"cache_read":0.2,"input":1.25,"output":2.5,"tiers":[{"cache_read":0.4,"input":2.5,"output":5,"tier":{"size":200000}}]},"description":"Grok model for agentic tool use, reasoning, coding, and live assistance","name":"Grok 4.20 Multi-Agent"},"grok-4.3":{"cost":{"cache_read":0.2,"input":1.25,"output":2.5,"tiers":[{"cache_read":0.4,"input":2.5,"output":5,"tier":{"size":200000}}]},"description":"xAI's Grok for chat, coding, agentic tools, and lower hallucination risk","name":"Grok 4.3"},"grok-4.5":{"cost":{"cache_read":0.3,"input":2,"output":6,"tiers":[{"cache_read":0.6,"input":4,"output":12,"tier":{"size":200000,"type":"context"}}]},"description":"xAI's latest Grok for chat, coding, agentic tools, and lower hallucination risk","name":"Grok 4.5"},"grok-build-0.1":{"cost":{"cache_read":0.2,"input":1,"output":2,"tiers":[{"cache_read":0.4,"input":2.0,"output":4.0,"tier":{"size":200000}}]},"description":"Fast Grok coding model tuned for agentic engineering and iterative edits","name":"Grok Build 0.1"},"grok-imagine-image":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Grok Imagine Image"},"grok-imagine-image-quality":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Grok Imagine Image Quality"},"grok-imagine-video":{"description":"Image model for prompt-driven generation, editing, and visual design workflows","name":"Grok Imagine Video"},"grok-imagine-video-1.5":{"description":"Video model for image-to-video generation, editing, and extension workflows","name":"Grok Imagine Video 1.5"}}},"zai":{"models":{"glm-4.5":{"cost":{"cache_read":0.11,"cache_write":0,"input":0.6,"output":2.2},"description":"Hybrid-reasoning GLM release that made the 4.5 line broadly useful","name":"GLM-4.5"},"glm-4.5-air":{"cost":{"cache_read":0.03,"cache_write":0,"input":0.2,"output":1.1},"description":"Lighter GLM-4.5 variant for fast coding assistance and cheaper agents","name":"GLM-4.5-Air"},"glm-4.5-flash":{"cost":{"cache_read":0,"cache_write":0,"input":0,"output":0},"description":"Efficient GLM model for fast reasoning, coding, and agent workflows","name":"GLM-4.5-Flash"},"glm-4.5v":{"cost":{"input":0.6,"output":1.8},"description":"GLM vision model for visual reasoning, documents, and multimodal agents","name":"GLM-4.5V"},"glm-4.6":{"cost":{"cache_read":0.11,"cache_write":0,"input":0.6,"output":2.2},"description":"Late GLM-4 workhorse for coding agents, reasoning, and structured tasks","name":"GLM-4.6"},"glm-4.6v":{"cost":{"input":0.3,"output":0.9},"description":"GLM vision model for visual reasoning, documents, and multimodal agents","name":"GLM-4.6V"},"glm-4.7":{"cost":{"cache_read":0.11,"cache_write":0,"input":0.6,"output":2.2},"description":"Mature GLM model for dependable coding, reasoning, and structured agent tasks","name":"GLM-4.7"},"glm-4.7-flash":{"cost":{"cache_read":0,"cache_write":0,"input":0,"output":0},"description":"Budget GLM lane for fast coding help, routing, and everyday automation","name":"GLM-4.7-Flash"},"glm-4.7-flashx":{"cost":{"cache_read":0.01,"cache_write":0,"input":0.07,"output":0.4},"description":"Efficient GLM model for fast reasoning, coding, and agent workflows","name":"GLM-4.7-FlashX"},"glm-5":{"cost":{"cache_read":0.2,"cache_write":0,"input":1.0,"output":3.2},"description":"General GLM flagship for coding, analysis, and tool-heavy engineering workflows","name":"GLM-5"},"glm-5-turbo":{"cost":{"cache_read":0.24,"cache_write":0,"input":1.2,"output":4.0},"description":"Faster GLM-5 lane for coding agents that need lower latency","name":"GLM-5-Turbo"},"glm-5.1":{"cost":{"cache_read":0.26,"cache_write":0,"input":1.4,"output":4.4},"description":"Strong GLM coding model for agentic engineering, terminals, and repository generation","name":"GLM-5.1"},"glm-5.2":{"cost":{"cache_read":0.26,"cache_write":0,"input":1.4,"output":4.4},"description":"Open flagship GLM for long-horizon coding agents and million-token context work","name":"GLM-5.2"},"glm-5v-turbo":{"cost":{"cache_read":0.24,"cache_write":0,"input":1.2,"output":4.0},"description":"Fast GLM vision model for screenshots, documents, and multimodal agent tasks","name":"GLM-5V-Turbo"}}}} diff --git a/raven/providers/litellm_provider.py b/raven/providers/litellm_provider.py index 68b6ee38..f4dac57f 100644 --- a/raven/providers/litellm_provider.py +++ b/raven/providers/litellm_provider.py @@ -13,9 +13,12 @@ import json_repair from loguru import logger +from raven.providers import prompt_cache from raven.providers.base import LLMProvider, LLMResponse, StreamDelta, ToolCallRequest from raven.providers.litellm_setup import import_litellm -from raven.providers.registry import find_by_keywords, find_by_model, find_gateway, split_model_id +from raven.providers.prompt_cache import CACHE_CONTROL +from raven.providers.registry import find_by_keywords, find_by_model, find_gateway +from raven.providers.wire import wire_model litellm = import_litellm() acompletion = litellm.acompletion @@ -102,6 +105,10 @@ def __init__( # Detect gateway / local deployment. # provider_name (from config key) is the primary signal; # api_key / api_base are fallback for auto-detection. + # Kept because the id alone cannot say where a request goes: a bare + # `anthropic/claude-...` sent through this client reads as Anthropic's + # wire, which is not the wire it will travel on. + self._provider_name = provider_name or "" self._gateway = find_gateway(provider_name, api_key, api_base) if self._gateway and self._gateway.name == "openrouter": self.extra_headers = {**_OPENROUTER_ATTRIBUTION, **self.extra_headers} @@ -147,53 +154,19 @@ def _strip_gateway_prefix(self, model: str) -> str: return model[len(prefix) :] if model.startswith(prefix) else model def _resolve_model(self, model: str) -> str: - """Resolve model name by applying provider/gateway prefixes.""" - if self._gateway: - # Gateway mode: apply gateway prefix, skip provider-specific prefixes. - # model_prefix, not the raw field: a gateway whose name is already - # LiteLLM's declares nothing, and reading the field would drop the - # prefix entirely -- sending the gateway's key to the vendor named in - # the model id. - prefix = self._gateway.model_prefix - if self._gateway.strip_model_prefix: - # One leading vendor segment, not everything but the last: a - # model id may itself contain a slash ("openai/gpt-oss-120b" is - # Groq's own name for it), and keeping only the tail truncated - # the id this gateway is asked to serve. - _, model = split_model_id(model) - if prefix and not model.startswith(f"{prefix}/"): - model = f"{prefix}/{model}" - return model - - # Standard mode: auto-prefix for known providers. - spec = find_by_model(model) - prefix = spec.model_prefix if spec else "" - if spec and prefix: - model = self._canonicalize_explicit_prefix(model, spec, prefix) - if not any(model.startswith(s) for s in (*spec.skip_prefixes, f"{prefix}/")): - model = f"{prefix}/{model}" + """The id this request is sent under. See ``providers.wire``.""" + return wire_model(model, gateway=self._gateway) - return model + def _supports_cache_control(self, model: str) -> bool: + """Return True when this request may carry cache_control blocks. - @staticmethod - def _canonicalize_explicit_prefix(model: str, spec: Any, canonical_prefix: str) -> str: - """Normalize an explicit prefix (`github-copilot/...`, a former name).""" - if "/" not in model: - return model - prefix, remainder = split_model_id(model) - if prefix not in spec.route_names: - return model - return f"{canonical_prefix}/{remainder}" + Decided by ``providers.prompt_cache``, which the token strategies ask too + -- three copies of this question disagreed, and the one here could not + have answered for the marks they place. + """ + from raven.providers.prompt_cache import accepts_cache_control - def _supports_cache_control(self, model: str) -> bool: - """Return True when the provider supports cache_control on content blocks.""" - if self._gateway is not None: - return self._gateway.supports_prompt_caching - # Keyword fallback for the same reason token_wise has one: an id routed - # through a vendor we carry no spec for ("bedrock/anthropic.claude-...") - # still reaches a model whose caching is the upstream vendor's. - spec = find_by_model(model) or find_by_keywords(model) - return spec is not None and spec.supports_prompt_caching + return accepts_cache_control(model, addressed_to=self._provider_name) def _apply_cache_control( self, @@ -206,10 +179,10 @@ def _apply_cache_control( if msg.get("role") == "system": content = msg["content"] if isinstance(content, str): - new_content = [{"type": "text", "text": content, "cache_control": {"type": "ephemeral"}}] + new_content = [{"type": "text", "text": content, "cache_control": CACHE_CONTROL}] else: new_content = list(content) - new_content[-1] = {**new_content[-1], "cache_control": {"type": "ephemeral"}} + new_content[-1] = {**new_content[-1], "cache_control": CACHE_CONTROL} new_messages.append({**msg, "content": new_content}) else: new_messages.append(msg) @@ -217,7 +190,7 @@ def _apply_cache_control( new_tools = tools if tools: new_tools = list(tools) - new_tools[-1] = {**new_tools[-1], "cache_control": {"type": "ephemeral"}} + new_tools[-1] = {**new_tools[-1], "cache_control": CACHE_CONTROL} return new_messages, new_tools @@ -323,8 +296,11 @@ async def chat( model = self._resolve_model(original_model) extra_msg_keys = self._extra_msg_keys(original_model, model) - if self._supports_cache_control(original_model) and not self.disable_auto_cache_control: - messages, tools = self._apply_cache_control(messages, tools) + if self._supports_cache_control(original_model): + if not self.disable_auto_cache_control: + messages, tools = self._apply_cache_control(messages, tools) + else: + messages, tools = prompt_cache.strip(messages, tools) # Clamp max_tokens to at least 1 — negative or zero values cause # LiteLLM to reject the request with "max_tokens must be at least 1". @@ -407,8 +383,11 @@ async def chat_stream( model = self._resolve_model(original_model) extra_msg_keys = self._extra_msg_keys(original_model, model) - if self._supports_cache_control(original_model) and not self.disable_auto_cache_control: - messages, tools = self._apply_cache_control(messages, tools) + if self._supports_cache_control(original_model): + if not self.disable_auto_cache_control: + messages, tools = self._apply_cache_control(messages, tools) + else: + messages, tools = prompt_cache.strip(messages, tools) max_tokens = max(1, max_tokens) @@ -442,26 +421,79 @@ async def chat_stream( kwargs["tools"] = tools kwargs["tool_choice"] = tool_choice or "auto" - response = await asyncio.wait_for(acompletion(**kwargs), self.generation.timeout) + def _retry_without_breakpoints(exc: Exception) -> bool: + """Learn the refusal and take the marks off, or say this is not one. + + The one retry this path takes. Restarting a *partially streamed* call + is the problem that kept retry out of here, and this is not that: the + refusal arrives before any chunk has been handed to the caller, so + nothing has been said that would have to be unsaid. Without it the + learned downgrade never reaches the surface that actually streams -- + the TUI, where the affected model answered 400 on every single turn. + """ + if prompt_cache.is_suppressed(original_model) or not prompt_cache.is_rejection(exc): + return False + prompt_cache.suppress(original_model) + kwargs["messages"], stripped = prompt_cache.strip(kwargs["messages"], kwargs.get("tools")) + if stripped is not None: + kwargs["tools"] = stripped + return True + + async def _open(): + return (await asyncio.wait_for(acompletion(**kwargs), self.generation.timeout)).__aiter__() + + async def _close(target: Any) -> None: + aclose = getattr(target, "aclose", None) + if aclose is not None: + await aclose() + # Per-chunk idle cap: the timer resets on every chunk, so a long but # steadily-progressing generation is fine while a mid-stream stall (no # bytes for `timeout` seconds) raises TimeoutError instead of hanging. - # aclose() in finally closes the underlying HTTP stream deterministically - # on that timeout, mirroring the `async with` cleanup on the other paths. - stream = response.__aiter__() + # Everything from the open onward sits inside the one try/finally, so the + # underlying HTTP stream is closed deterministically on any exit -- a + # first-chunk timeout included, which is the most likely one there is + # (gateway queueing, cold start). + # A chunk of None is a chunk, not the end of the stream. Pulling the + # first one before the loop needs a value meaning "there was none", and + # reusing None for it would let a provider that yields one truncate the + # response silently -- which is not what the loop did before. + done = object() + + stream: Any = None try: - while True: + # The open and the first pull are one unit, and the `except` has to + # cover both: an OpenAI-shaped route raises at the open, and a gateway + # that defers the request until the first pull raises there instead. + try: + stream = await _open() + first = await asyncio.wait_for(stream.__anext__(), self.generation.timeout) + except StopAsyncIteration: + first = done + except Exception as exc: + if not _retry_without_breakpoints(exc): + raise + # The refused stream is finished with; closing it before opening + # the replacement keeps at most one live at a time. It is None + # when the open itself was what failed. + await _close(stream) + stream = await _open() try: - chunk = await asyncio.wait_for(stream.__anext__(), self.generation.timeout) + first = await asyncio.wait_for(stream.__anext__(), self.generation.timeout) except StopAsyncIteration: - break + first = done + + chunk = first + while chunk is not done: delta = self._normalize_stream_chunk(chunk) if delta is not None: yield delta + try: + chunk = await asyncio.wait_for(stream.__anext__(), self.generation.timeout) + except StopAsyncIteration: + break finally: - aclose = getattr(stream, "aclose", None) - if aclose is not None: - await aclose() + await _close(stream) def _normalize_stream_chunk(self, chunk: Any) -> StreamDelta | None: """Normalize a raw provider chunk into a StreamDelta. diff --git a/raven/token_wise/model_catalog_cache.py b/raven/providers/model_catalog_cache.py similarity index 97% rename from raven/token_wise/model_catalog_cache.py rename to raven/providers/model_catalog_cache.py index e7b46e85..7f5d050b 100644 --- a/raven/token_wise/model_catalog_cache.py +++ b/raven/providers/model_catalog_cache.py @@ -9,7 +9,7 @@ Named after what it persists (the model catalog), not its source: the storage layer is source-agnostic, so a future catalog source reuses it unchanged. This is the storage layer only; freshness (TTL), the in-process tier, and the actual -fetch are the caller's concern (see ``pricing._fetch_openrouter_models``). +fetch are the caller's concern (see ``providers.rates._fetch_openrouter_models``). """ from __future__ import annotations diff --git a/raven/providers/openai_codex_provider.py b/raven/providers/openai_codex_provider.py index 97cef197..39f58edd 100644 --- a/raven/providers/openai_codex_provider.py +++ b/raven/providers/openai_codex_provider.py @@ -109,9 +109,15 @@ def get_default_model(self) -> str: def _strip_model_prefix(model: str) -> str: - if model.startswith("openai-codex/") or model.startswith("openai_codex/"): - return model.split("/", 1)[1] - return model + """The id the Responses API is asked for. See ``providers.wire``. + + The stored id names this provider so nothing else can claim it; the backend + knows only the vendor's own slug. + """ + from raven.providers.registry import find_by_name + from raven.providers.wire import wire_model + + return wire_model(model, spec=find_by_name("openai_codex")) def _build_headers(account_id: str, token: str) -> dict[str, str]: diff --git a/raven/providers/pin.py b/raven/providers/pin.py new file mode 100644 index 00000000..4f2c1406 --- /dev/null +++ b/raven/providers/pin.py @@ -0,0 +1,113 @@ +"""Which provider ``agents.defaults.provider`` should name after a model change. + +The pin overrides what a model id says, so a stale one silently sends the new +model's request to the old vendor -- with the old vendor's key. The rule lives +here so the TUI picker and the CLI answer the same way, rather than the CLI +telling the user to edit the field by hand. + +The four cases, and why each is what it is: + +====================== ========================================================== +a provider was named that provider. The picker always sends one, and a + ``--provider`` flag is the user saying it outright. +the id names a vendor that vendor's spec. The id is evidence and it is +we carry a spec for unambiguous. +the id is prefixed but ``auto``. Keeping the old pin would hand its key to a +we have no spec for it vendor it does not belong to; auto-detection is the + honest answer for "somebody, not necessarily who you had". +a bare id ask the pinned provider whether it serves this model, + and keep the pin only if it does -- see + ``resolve_bare_against_pin``. +====================== ========================================================== +""" + +from __future__ import annotations + +AUTO = "auto" + + +def resolve(model: str, *, provider: str = "", pinned: str = "") -> str | None: + """The provider to pin for ``model``, or None when nothing can be told. + + ``provider`` is an explicitly chosen one (a picker selection, a ``--provider`` + flag); ``pinned`` is what ``agents.defaults.provider`` currently holds. + None means the caller must ask rather than guess -- there is no answer that + does not risk sending one vendor's key to another. + """ + from raven.providers.registry import find_by_model + + if provider: + return provider + + spec = find_by_model(model) + if spec is not None: + # Only if that vendor is actually configured. A pin is consulted before + # anything else and answers with that vendor's section whether or not it + # holds credentials, so pinning an unconfigured one means every request + # fails on a missing key -- never reaching the fallback that lets a + # gateway serve a model whose id names the vendor behind it. `auto` is + # not the weaker answer here; it is the one that reaches the gateway. + return spec.name if _is_configured(spec.name) else AUTO + if "/" in model: + # A prefixed id whose vendor has no spec of ours ("mistral/..."): keeping + # the previously forced provider would send that provider's key to this + # other vendor, so hand routing back to auto-detection. + return AUTO + return resolve_bare_against_pin(model, pinned=pinned) + + +def _is_configured(provider: str) -> bool: + """Does this provider have a section holding usable credentials? + + Takes a registered name -- the only caller passes a spec's. An unregistered + one raises out of ``get_provider_config`` rather than answering "no", which + for a misspelling is the difference between a loud failure and quietly + routing somewhere else. + """ + from raven.config.update_providers import get_provider_config + from raven.providers.auth import credential_status + + section = get_provider_config(provider, redact_secrets=False) + return credential_status(provider, section, include_external=True).ok + + +def resolve_bare_against_pin(model: str, *, pinned: str) -> str | None: + """Who serves this bare id, when its own text names nobody? + + A bare id that matches no provider's keywords is what a vendor Raven holds no + spec for looks like, and the picker never produces one -- it always sends the + provider alongside. So this is the hand-typed path, where the only other + evidence is the provider currently pinned. + + Rather than guess, ask whether that provider serves the model: its own curated + list first, then the catalogue. If it does, the pin was right and stays. A local + deployment stays too without asking -- its server names whatever models it + likes, and there is no key to mis-route. + + With no such evidence the pin is not kept: it would send one vendor's key to + another, the mis-routing the prefix rules exist to prevent. Returning None + leaves the caller to say so rather than pick a vendor on the user's behalf. + """ + from raven.config.update_providers import get_provider_config + from raven.providers.common_models import common_models_for, litellm_models_for + from raven.providers.registry import find_by_name, split_model_id + + if not pinned or pinned == AUTO: + return AUTO + + spec = find_by_name(pinned) + if spec is not None and spec.is_local: + return pinned + + try: + configured = get_provider_config(pinned, redact_secrets=True).get("models") or [] + except KeyError: + configured = [] + for candidate in (*configured, *common_models_for(pinned), *litellm_models_for(pinned)): + # Stripping the prefix covers every spelling the sources use: the + # catalogue keys ids the way LiteLLM spells the vendor, a hand-added one + # sits in the provider's list bare. + _, bare = split_model_id(candidate) + if bare == model or candidate == model: + return pinned + return None diff --git a/raven/providers/prompt_cache.py b/raven/providers/prompt_cache.py new file mode 100644 index 00000000..3a10312a --- /dev/null +++ b/raven/providers/prompt_cache.py @@ -0,0 +1,210 @@ +"""Whether a request may carry Anthropic-shaped ``cache_control`` breakpoints. + +One question, asked from three places -- the provider that builds the request, +and the two token strategies that place breakpoints before it. It used to be +answered by a copy of the same function in each, which is how the copies came to +disagree: the provider's only ever marked the system message and the tool list, +so fixing it there could not have changed what the strategies stamp onto the last +conversation message, and that is where the doubling came from. + +The answer is **(wire x model family)**, not the wire alone: + +* the wire has to have somewhere to put the field. An OpenAI-shaped API does + not, and a gateway speaks its own shape regardless of who it fronts, so + ``ProviderSpec.supports_prompt_caching`` is asked of whatever the request is + actually addressed to. +* the model has to be one whose vendor reads it. A gateway accepts the field for + every model it fronts and forwards it to vendors that do not: it is then billed + as an unrecognized block rather than refused, which doubles a prompt silently. + +Suppression is the fourth answer, and it is learned rather than declared, +because no table here can predict it. See ``suppress``. +""" + +from __future__ import annotations + +import re +from typing import Any + +from loguru import logger + +CACHE_CONTROL: dict[str, str] = {"type": "ephemeral"} + +#: The vendor whose API defines this field. A model reaches it directly or +#: through a gateway; either way it is the one that reads the breakpoints. +_DIALECT_OWNER = "anthropic" + +#: (provider, model) pairs an upstream rejected the field for, learned at +#: runtime. Process-local on purpose -- see ``suppress``. +_SUPPRESSED: set[str] = set() + + +def accepts_cache_control(model: str, *, addressed_to: str = "") -> bool: + """May this request carry ``cache_control`` blocks? + + False for an empty id, for a wire with nowhere to put the field, for a model + whose vendor does not read it, and for anything an upstream has already + rejected it for. + + ``addressed_to`` is the provider actually serving the request, for the one + caller that knows it independently of the id. A stored id names its provider, + so the two normally agree -- but a bare ``anthropic/claude-...`` handed to a + SiliconFlow client reads as Anthropic's wire from the id alone, and that wire + has nowhere to put the field. Passing it keeps the answer about the request + rather than about the string. + """ + if not model or model in _SUPPRESSED: + return False + + from raven.providers.registry import find_by_keywords, find_by_model, find_by_name + + addressed = find_by_name(addressed_to) if addressed_to else find_by_model(model) + if addressed is None or not addressed.supports_prompt_caching: + return False + + # The family cannot be read off the id's prefixes: the leading one names the + # gateway, and the upstream segment is spelled the gateway's way ("google", + # not "gemini"). So it is read from the id's keywords. A direct route + # answers the same way: `anthropic/claude-...` matches on both. + family = find_by_keywords(model) + return family is not None and family.name == _DIALECT_OWNER + + +def suppress(model: str) -> None: + """Stop sending ``cache_control`` for this model for the rest of the process. + + Called when an upstream has answered a marked request with a rejection. The + case this exists for cannot be predicted from any table: OpenRouter routes + ``anthropic/claude-3-haiku`` to Amazon Bedrock, whose dialect is + ``cachePoint``, and neither OpenRouter's catalogue nor LiteLLM's says so -- + both correctly report that the model caches. + + Deliberately not persisted. Upstream routing is a runtime decision that + changes, so a file written tonight would still be answering next month; the + cost of forgetting is one extra request per model per process, and the cost + of a stale file is caching silently switched off for a model that regained + it. + """ + if model and model not in _SUPPRESSED: + _SUPPRESSED.add(model) + logger.info("prompt cache: {} rejected cache_control upstream; not sending it again", model) + + +def is_suppressed(model: str) -> bool: + return model in _SUPPRESSED + + +def reset_suppressions() -> None: + """Only useful for tests -- production learns and keeps.""" + _SUPPRESSED.clear() + + +#: How a client spells "the request was refused". Not redundant with the status +#: below: a gateway paraphrasing its upstream can drop the numeric code entirely +#: ("Bad Request: ... did not allow prompt caching"), and the spelling that +#: reaches us carries a space, which the run-together forms do not match. +_BAD_REQUEST_MARKERS = ("bad request", "badrequest", "bad_request", "invalid_request") + +#: The status as its own token. As a bare substring it also matched the "400" in +#: "retry after 1400ms", so a rate limit or a timeout whose text happened to name +#: the field read as a refusal -- and the cost of that is caching switched off +#: for the model, quietly, for the rest of the process. +_STATUS_400 = re.compile(r"\b400\b") + +#: How a refusal names itself. More than the field name, because a gateway +#: paraphrases its upstream: a Bedrock refusal reaches us saying only "did not +#: allow prompt caching", with the field name nowhere in the text. +_REFUSAL_MARKERS = ("cache_control", "prompt caching", "prompt_caching") + + +#: Where a client keeps the response body when its ``str()`` is a summary. +#: LiteLLM's streaming path raises ``MaskedHTTPStatusError``, whose text is +#: "Client error '400 Bad Request' for url ..." and names nothing at all; the +#: body it was built from sits on these attributes. Reading only ``str(exc)`` +#: made the same refusal learnable on one path and invisible on the other. +_BODY_ATTRS = ("text", "message", "body") + + +def _searchable(error: object) -> str: + """Everything this error says about itself, lowercased. + + Both layers paraphrase: the gateway paraphrases the upstream, and the client + paraphrases the gateway, so the body has to be gathered off the exception + rather than read out of ``str()``. + """ + if isinstance(error, str): + return error.lower() + parts = [str(error)] + parts.extend(str(getattr(error, attr, "") or "") for attr in _BODY_ATTRS) + return " ".join(parts).lower() + + +def is_rejection(error: object) -> bool: + """Is this error the upstream refusing prompt-cache breakpoints? + + Takes the exception rather than a rendered string, because which of the two + carries the body depends on the path: the non-streaming call raises one whose + ``str()`` includes it, and the streaming call raises one whose ``str()`` is a + URL and a status. + + Narrow on purpose: both a name for what was refused and a refused-request + marker are required. Naming alone would read a timeout whose payload was + logged as a dialect problem and switch caching off for the rest of the + process; the status alone would swallow every other way a request can be + malformed into a silent retry. + + Nothing is swallowed either way -- the retry sends the same request without + the field, and if that fails too the second error surfaces unchanged. + """ + text = _searchable(error) + if not any(name in text for name in _REFUSAL_MARKERS): + return False + return bool(_STATUS_400.search(text)) or any(m in text for m in _BAD_REQUEST_MARKERS) + + +def strip( + messages: list[dict[str, Any]], + tools: list[dict[str, Any]] | None, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]] | None]: + """Copies of ``messages`` and ``tools`` with every breakpoint removed. + + The last word belongs to whoever sends the request. A token strategy sees + only the model id, and an id can name a vendor the request is not going to: + `anthropic/claude-3` served through an OpenAI-shaped gateway is marked by the + strategy and then refused -- or, worse, quietly billed twice -- by the wire it + actually travels on. The client knows its own destination, so it is the one + that takes off what should not go. + + Needed as well as ``suppress``, not instead of it: the strategies place their + breakpoints upstream of the provider, so by the time a request has failed the + marks are already in the payload the retry would resend. Suppression stops + the provider from adding its own on the way back out; this takes off the ones + that are already there. + """ + return [_strip_message(m) for m in messages], _strip_blocks(tools) + + +def _strip_message(message: dict[str, Any]) -> dict[str, Any]: + cleaned = {k: v for k, v in message.items() if k != "cache_control"} + content = cleaned.get("content") + if isinstance(content, list): + blocks = _strip_blocks(content) or [] + # Undoing the key is not undoing the marking. To have somewhere to put a + # breakpoint, the strategy rewrites string content into a one-element + # text block -- so removing the field alone still sends an + # Anthropic-shaped payload to a wire that was just judged unable to carry + # one, and "content must be a string" is among the commonest ways an + # OpenAI-compatible endpoint refuses. That refusal names neither the + # field nor prompt caching, so nothing learns from it either. + only = blocks[0] if len(blocks) == 1 and isinstance(blocks[0], dict) else None + if only is not None and set(only) == {"type", "text"} and only["type"] == "text": + cleaned["content"] = only["text"] + else: + cleaned["content"] = blocks + return cleaned + + +def _strip_blocks(blocks: list[dict[str, Any]] | None) -> list[dict[str, Any]] | None: + if blocks is None: + return None + return [{k: v for k, v in b.items() if k != "cache_control"} if isinstance(b, dict) else b for b in blocks] diff --git a/raven/providers/rates.py b/raven/providers/rates.py new file mode 100644 index 00000000..50a2d199 --- /dev/null +++ b/raven/providers/rates.py @@ -0,0 +1,508 @@ +"""What a model costs per token and how much context it takes. + +Both are facts about a provider's catalogue, so they are decided here and not by +whoever is about to report a number. They used to live in ``token_wise.pricing`` +next to the cost formula, which put a provider decision outside +``raven.providers`` -- and a decision outside its module grows a second copy: the +benchmark runner carried its own rate table, and the window resolution grew an +OpenRouter fallback that answered for vendors OpenRouter does not serve. + +Two questions, deliberately answered from different places: + +* **rates** price a call after it happened. A wrong figure costs an inaccurate + total, so the ladder can reach for a community-maintained catalogue. +* **the context window** sizes trimming, so it shapes the next request. Only the + tables that also route may answer it, and an unknown window is answered with + ``None`` -- the caller keeps its configured default, which is honest, where a + window borrowed from another vendor is silently wrong. +""" + +from __future__ import annotations + +import pathlib +import threading +import time +from functools import lru_cache + +import httpx +from loguru import logger + +from raven.providers import model_catalog_cache + +#: Rate pair: (prompt_cost_per_token, completion_cost_per_token) in USD. +#: Keep this table small -- it is a fallback for brand-new models that LiteLLM +#: has not indexed yet. Check LiteLLM first before adding here. +_FALLBACK_PRICING: dict[str, tuple[float, float]] = { + # OpenRouter model pages (snapshot 2026-03) + "z-ai/glm-4.5-air": (0.13e-6, 0.85e-6), # $0.13/$0.85 per 1M +} + +# Live OpenRouter price table, fetched lazily and cached 1h in-process. +_OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" +_OPENROUTER_CACHE_TTL = 3600 +_OPENROUTER_CACHE: dict[str, dict] = {} +_OPENROUTER_CACHE_TIME: float = 0.0 +# Monotonic stamp of the last background warm attempt (0 = never), and how +# long a failed one waits before another is allowed. See +# warm_catalog_in_background. +_WARM_AT: float = 0.0 +_WARM_RETRY_SECONDS = 300.0 + + +def _litellm_price_table() -> dict: + """LiteLLM's static price table, or an empty dict if it cannot be imported.""" + try: + from raven.providers.litellm_setup import import_litellm + + return getattr(import_litellm(), "model_cost", None) or {} + except Exception: + return {} + + +def _table_entry(model: str) -> dict | None: + """The table row keyed exactly by this model id, or None. + + Deliberately no prefix-stripping fallback. It looked like a free replacement + for asking LiteLLM, but the ask does more than key normalization: for an + "openrouter//" candidate it derives OpenRouter's own numbers, + which are in no table row -- and stripping to the direct row answered three + MiniMax models with the direct figure where LiteLLM had reported OpenRouter's. + The table is read here to skip the ask where the ask cannot be made; it does + not replace it. + """ + entry = _litellm_price_table().get(model) + return entry if isinstance(entry, dict) else None + + +@lru_cache(maxsize=1) +def _drivers_dir() -> pathlib.Path | None: + """Where the installed LiteLLM keeps its per-provider drivers.""" + try: + from raven.providers.litellm_setup import import_litellm + + return pathlib.Path(import_litellm().__file__).parent / "llms" + except Exception: + return None + + +def _may_prompt(model: str) -> bool: + """Would handing this model to LiteLLM start an interactive login? + + Three of its drivers ship a device-flow authenticator, and every entry point + that resolves a model reaches it -- ``get_model_info``, ``cost_per_token`` and + ``validate_environment`` alike. With no token file the call prints a device + code to stdout and blocks. Any segment counts, not just the first: a bare id + and its ``openrouter/`` alias reach the same driver, so checking only the + head lets the second candidate hang the lookup anyway. + + Asked of the installed package rather than a snapshot of it -- a driver is one + that ships ``authenticator.py``. A frozen list would have to be regenerated on + every LiteLLM bump, and a stale one brings the hang back for the vendor it + missed; this cannot go stale. The check is a stat, and the callers have already + paid for the import. + """ + drivers = _drivers_dir() + if drivers is None: + return False + return any((drivers / part / "authenticator.py").exists() for part in model.split("/") if part) + + +def _numeric(entry: dict | None, *fields: str) -> float | None: + """First numeric value among ``fields``, or None. + + The table ships a self-documenting sample row whose numeric-looking fields + hold prose, so the type check is load-bearing rather than defensive. + """ + if not isinstance(entry, dict): + return None + for field in fields: + value = entry.get(field) + if isinstance(value, (int, float)) and value: + return float(value) + return None + + +def is_plan_billed(model: str) -> bool: + """Is this model's provider billed by subscription rather than per token? + + Asked wherever a dollar figure is about to be reported, because on a + subscription there is no per-token figure to report -- not even zero, which + reads as free. + """ + from raven.providers.registry import find_by_model + + spec = find_by_model(model) + return bool(spec and spec.billing == "plan") + + +def _candidates(model: str) -> list[str]: + """Every id LiteLLM's table might file this model under. See ``providers.wire``.""" + from raven.providers.wire import metadata_candidates + + return metadata_candidates(model) + + +def _try_litellm_rates(model: str, input_tokens: int, output_tokens: int) -> tuple[float, float] | None: + """Ask LiteLLM for per-token rates. Returns (prompt_rate, completion_rate) or None.""" + try: + from raven.providers.litellm_setup import import_litellm + + litellm = import_litellm() + except Exception: + return None + + # litellm.cost_per_token expects *at least* 1 non-zero token to compute. + # We pass synthetic tokens to recover the per-token rate. + probe_in = input_tokens if input_tokens else 1 + probe_out = output_tokens if output_tokens else 1 + + for candidate in _candidates(model): + if _may_prompt(candidate): + # Skipped, not read from the table: the rows these families have are + # priced at zero, which this function already treats as unknown, so + # reading them would add a branch that cannot fire. The caller falls + # through to the remaining tiers, which is what any model LiteLLM does + # not price already does. + continue + try: + prompt_cost, completion_cost = litellm.cost_per_token( + model=candidate, prompt_tokens=probe_in, completion_tokens=probe_out + ) + except Exception: + continue + if prompt_cost is None or completion_cost is None: + continue + if prompt_cost == 0 and completion_cost == 0: + # LiteLLM returns (0, 0) when the model is unknown -- treat as miss. + continue + return prompt_cost / probe_in, completion_cost / probe_out + + return None + + +def _fetch_openrouter_models() -> dict[str, dict]: + """Return OpenRouter's model table, fetched live and cached 1h in-process. + + Each entry is ``{"pricing": ..., "context_length": ...}``, keyed by the full + id. On any network failure, returns the stale cache (or an empty dict) -- + pricing must never raise into the cost path. + """ + global _OPENROUTER_CACHE, _OPENROUTER_CACHE_TIME + + now = time.time() + if _OPENROUTER_CACHE and (now - _OPENROUTER_CACHE_TIME) < _OPENROUTER_CACHE_TTL: + return _OPENROUTER_CACHE + + # Disk tier: warm-start (or pick up a sibling process's fresher fetch) + # from a fresh on-disk cache without touching the network. + disk = model_catalog_cache.load() + if disk is not None and (now - disk[1]) < _OPENROUTER_CACHE_TTL: + _OPENROUTER_CACHE, _OPENROUTER_CACHE_TIME = disk + return _OPENROUTER_CACHE + + try: + with httpx.Client(timeout=10.0) as client: + resp = client.get(_OPENROUTER_MODELS_URL) + resp.raise_for_status() + data = resp.json() + except Exception as exc: + logger.debug("rates: OpenRouter models fetch failed ({}), degrading", exc) + if _OPENROUTER_CACHE: + return _OPENROUTER_CACHE + if disk is not None: + _OPENROUTER_CACHE, _OPENROUTER_CACHE_TIME = disk + return _OPENROUTER_CACHE + return {} + + cache: dict[str, dict] = {} + for model in data.get("data", []): + model_id = model.get("id", "") + if not model_id: + continue + arch = model.get("architecture") or {} + mods = arch.get("input_modalities") + entry = { + "pricing": model.get("pricing") or {}, + "context_length": model.get("context_length"), + # What the model accepts as input ("text" / "image" / "audio" / + # "file" / "video"). The catalog is fetched for prices, and it is + # also the only published answer to "can this model see" that + # states itself for every model it lists -- see + # ``capabilities.supports_vision``. + "input_modalities": list(mods) if isinstance(mods, list) and mods else None, + } + cache[model_id] = entry + if "/" in model_id: + cache.setdefault(model_id.split("/", 1)[1], entry) + + _OPENROUTER_CACHE = cache + _OPENROUTER_CACHE_TIME = time.time() + model_catalog_cache.save(cache) + return cache + + +def warm_catalog_in_background() -> None: + """Start filling the catalog off the request path, without blocking a turn. + + The pricing path cannot be relied on to do it. It asks LiteLLM's static + table first and only reaches this catalog when that table *misses*, so for + every model LiteLLM does carry -- which is every model Raven ships a default + for -- the catalog is never fetched and a reader like + :func:`openrouter_input_modalities` has nothing to read, forever. + + Called instead of fetching inline because the fetch is synchronous with a + 10s timeout: on a machine that cannot reach the host, doing it in the turn + would stall the turn. A cold caller therefore degrades until the fetch lands. + + Retried on a cooldown rather than attempted once. An attempt that fails + proves nothing about the next one -- the first turn of a session routinely + runs before a VPN is up or a proxy has authenticated -- and a single latched + attempt would leave the reader answering from an empty catalog for the whole + process. A success needs no cooldown: the filled cache is itself the guard. + """ + global _WARM_AT + + if _OPENROUTER_CACHE: + return + now = time.monotonic() + if _WARM_AT and now - _WARM_AT < _WARM_RETRY_SECONDS: + return + _WARM_AT = now + + # Resolved here rather than inside the thread. A thread body that looks the + # name up on entry can lose a race with whoever patched it -- a test seam + # restored between ``start()`` and the thread's first bytecode would send a + # real request from inside the suite and write the real cache file. + fetch = _fetch_openrouter_models + + def _run() -> None: + try: + fetch() + except Exception as exc: # the fetch degrades internally; a thread must not die loudly + logger.debug("rates: background catalog warm failed ({})", exc) + + threading.Thread(target=_run, name="raven-model-catalog-warm", daemon=True).start() + + +def _cached_catalog_only() -> dict[str, dict]: + """Whatever catalog is already in hand, at any age, without fetching. + + ``_fetch_openrouter_models`` is synchronous with a one-hour TTL, so calling + it from a request path would hand one turn a stall whenever the hour rolls + over. Prices are why that TTL is short; a model's input modalities are not, + so this reader takes a stale table happily and an absent one as "no answer". + Filling an absent one is :func:`warm_catalog_in_background`'s job. + """ + global _OPENROUTER_CACHE + + if _OPENROUTER_CACHE: + return _OPENROUTER_CACHE + disk = model_catalog_cache.load() + if disk is None: + return {} + # Re-checked after the read, not just before it: ``load()`` touches the + # filesystem and releases the GIL, so a background warm can land in that + # window with both a fresher table and a fresh ``_OPENROUTER_CACHE_TIME``. + # Overwriting it with this stale copy would leave that timestamp vouching for + # the wrong table, and the fetch's TTL check would then skip the refetch. + if _OPENROUTER_CACHE: + return _OPENROUTER_CACHE + # Kept so the next lookup does not re-read and re-parse the file. + # ``_OPENROUTER_CACHE_TIME`` is deliberately left alone: the fetch reads it + # to decide freshness, and this table is of unknown age -- good enough for a + # modality question, not to be mistaken for fresh pricing. + _OPENROUTER_CACHE = disk[0] + return _OPENROUTER_CACHE + + +def openrouter_input_modalities(model: str) -> tuple[str, ...] | None: + """What the catalog says ``model`` accepts as input, or ``None``. + + ``None`` means the catalog has no entry (or one written before this field + was kept), never "text only": this source states itself for every model it + lists, so silence is absence rather than a denial. + + Matched on the full id and then the bare alias, case-folded -- the catalog + spells every id it publishes in lower case, while a routed id need not + (``minimax/MiniMax-M2``). Punctuation is *not* normalized away, and that + restraint is the point: an id that survives only a fuzzier match is an id + this catalog does not actually list, and the only thing a wrong match can do + here is deny vision to a model that has it. ``azure/`` and the local + runtimes take a user-chosen deployment or tag name where every other + provider takes a model id, so ``azure/gpt4`` and ``ollama/phi4`` would join + against ``openai/gpt-4`` and ``microsoft/phi-4`` on a punctuation-stripping + key and lose every picture, silently, on a deployment that may well serve a + vision model. Losing the fuzzy tier costs nothing measurable: on the live + catalog every model it additionally matched either already answers "can see" + (the default when there is no answer at all) or is one of these false + denials. + + Reads only what is already cached -- see :func:`_cached_catalog_only`. + """ + key = model.removeprefix("openrouter/").lower() + table = _cached_catalog_only() + entry = table.get(key) + if entry is None and "/" in key: + entry = table.get(key.split("/", 1)[1]) + if not entry: + return None + mods = entry.get("input_modalities") + return tuple(mods) if isinstance(mods, list) and mods else None + + +def _lookup_openrouter_entry(model: str) -> dict | None: + """This model's row in OpenRouter's catalogue, or None. + + Only for ids that name OpenRouter. The table was once consulted for every id, + which reads across vendors: a self-hosted ``hosted_vllm/qwen3-32b`` matched + OpenRouter's ``qwen/qwen3-32b`` and was reported at a price and a context + window belonging to somebody else's deployment. What made it wrong was asking + this table about a request that does not go to OpenRouter -- not the bare + alias, which stays because within OpenRouter's own namespace a bare id names + the same model the full one does. + """ + if not model.startswith("openrouter/"): + return None + key = model.removeprefix("openrouter/") + table = _fetch_openrouter_models() + entry = table.get(key) + if entry is None and "/" in key: + entry = table.get(key.split("/", 1)[1]) + return entry + + +def _try_openrouter_rates(model: str) -> tuple[float, float] | None: + """Look up live OpenRouter per-token rates. Returns rates or None.""" + entry = _lookup_openrouter_entry(model) + if not entry: + return None + pricing = entry.get("pricing") or {} + try: + return float(pricing["prompt"]), float(pricing["completion"]) + except (KeyError, TypeError, ValueError): + return None + + +def _try_snapshot_rates(model: str) -> tuple[float, float] | None: + """The vendor's own published price, from the bundled models.dev snapshot. + + Reaches vendors LiteLLM has not indexed without reading another vendor's + row: the snapshot is keyed by provider, so a direct ``zai/glm-5.2`` is + answered by Z.ai's figure rather than by whatever OpenRouter charges for a + model with a similar name. Costs are published per million tokens. + """ + from raven.providers.catalog import model_cost + + cost = model_cost(model) + if not cost: + return None + prompt = _numeric(cost, "input") + completion = _numeric(cost, "output") + if prompt is None or completion is None: + return None + return prompt / 1e6, completion / 1e6 + + +def token_rates(model: str, input_tokens: int = 0, output_tokens: int = 0) -> tuple[float, float] | None: + """This model's (prompt, completion) cost per token in USD, or None. + + Ladder, most authoritative first: + + 1. LiteLLM's own table -- it also routes the request, so its answer and the + call agree by construction; + 2. OpenRouter's live catalogue, and only for ids that name OpenRouter. Ahead + of the snapshot because for those ids OpenRouter is the party doing the + billing, and its table is current where a bundled copy is from whenever it + was refreshed; + 3. the bundled models.dev snapshot, keyed by provider, which reaches vendors + LiteLLM has not indexed without reading another vendor's row; + 4. the manual table above, for a model too new for all three. + + Tier 1 carries a deliberate exception to "only an id naming OpenRouter reads + OpenRouter": ``wire.metadata_candidates`` offers LiteLLM the ``openrouter/`` + alias of a direct id as a *second* candidate, after the vendor's own row. + That is safe where tiers 2 and 3 are not, because LiteLLM is the thing doing + the sending -- it is answering about a request it would make, not reading a + stranger's catalogue. The direct row is asked first for the same reason: + asked alias-first, one model reported half its window at half its price. + + A bare id -- no prefix -- reaches tier 1 and tier 4 only. Tiers 2 and 3 are + both keyed by vendor, and matching a bare name across either is what priced a + self-hosted deployment at a hosted model's rate. A bare id left by an older + version is priced as unknown until its model is picked again, which stores it + qualified. + + Token counts are passed through because a vendor may price by size, so the + rate for a 200k-token prompt is not always the rate for a short one. + """ + return ( + _try_litellm_rates(model, input_tokens, output_tokens) + or _try_openrouter_rates(model) + or _try_snapshot_rates(model) + or _FALLBACK_PRICING.get(model.removeprefix("openrouter/")) + ) + + +def _try_litellm_context_window(model: str) -> int | None: + """LiteLLM's static model metadata -- offline, covers most mapped providers.""" + try: + from raven.providers.litellm_setup import import_litellm + + litellm = import_litellm() + except Exception: + return None + + for candidate in _candidates(model): + # The table before the ask: it holds every model the interactive-login + # drivers are asked about in practice, and reading it cannot prompt. + window = _numeric(_table_entry(candidate), "max_input_tokens", "max_tokens") + if window: + return int(window) + if _may_prompt(candidate): + continue + try: + info = litellm.get_model_info(candidate) + except Exception: + continue + window = _numeric(info, "max_input_tokens", "max_tokens") + if window: + return int(window) + return None + + +def resolve_context_window(model: str) -> int | None: + """Return a model's real context window in tokens, or None. + + LiteLLM's static metadata first, then OpenRouter's catalogue for ids that + name OpenRouter. The snapshot is deliberately not a source: a window sizes + trimming, so a community-maintained file that goes stale or wrong would + shape the next request rather than cost a label. Unknown models return None + so the caller keeps its configured default. + """ + window = _try_litellm_context_window(model) + if window: + return window + + entry = _lookup_openrouter_entry(model) + if entry: + try: + length = int(entry.get("context_length") or 0) + except (TypeError, ValueError): + length = 0 + if length: + return length + return None + + +def reset_openrouter_cache() -> None: + """Clear the in-process OpenRouter catalog cache. + + Only useful for tests -- pair it with the ``model_catalog_cache._CACHE_PATH`` + seam to exercise the disk tiers without touching the real ~/.raven/cache/. + """ + global _OPENROUTER_CACHE, _OPENROUTER_CACHE_TIME, _WARM_AT + _OPENROUTER_CACHE = {} + _OPENROUTER_CACHE_TIME = 0.0 + # Reset too, or a warm attempt from an earlier test leaves this one on a + # cooldown it never asked for. + _WARM_AT = 0.0 diff --git a/raven/providers/registry.py b/raven/providers/registry.py index e22aee8a..b0bebf4e 100644 --- a/raven/providers/registry.py +++ b/raven/providers/registry.py @@ -638,24 +638,6 @@ def litellm_spelling(name: str | None) -> str: return wanted -#: Providers whose own client strips the prefix back off before use, so a model id -#: may -- and must -- carry the name that resolves to them. Everyone else either -#: routes on it (LiteLLM does the stripping) or uses the id verbatim: Azure puts it -#: in a URL path as a deployment name, where a prefix would become part of the path. -_PREFIX_IS_PUBLIC_ONLY = frozenset({"minimax_global", "minimax_cn", "openai_codex"}) - - -def needs_public_model_prefix(spec: "ProviderSpec | None") -> bool: - """Must a model id for this provider be written with its own name in front? - - Written bare it is claimed by keyword matching instead -- "gpt-5.6-sol" - resolves to OpenAI -- and the request goes to a provider that does not serve - it. Every surface that stores a model id asks this, so the answer is here - rather than in each of them. - """ - return spec is not None and spec.name in _PREFIX_IS_PUBLIC_ONLY - - def public_model_prefix(spec: "ProviderSpec") -> str: """The prefix a user writes to reach THIS provider, which is not always the one that goes on the wire. diff --git a/raven/providers/wire.py b/raven/providers/wire.py new file mode 100644 index 00000000..ad9e1847 --- /dev/null +++ b/raven/providers/wire.py @@ -0,0 +1,224 @@ +"""Storage form to wire form: the one place a stored model id becomes a sent one. + +A model id is written down in one shape and sent in another. `openrouter/x` is +stored with its gateway named so the picker and the router agree on who serves +it, but MiniMax's own client wants the prefix gone, LiteLLM wants the vendor it +routes on in front, and Azure wants the bare deployment name because the id +lands in a URL path. + +That conversion used to be spelled at each client. The spellings drifted: the +standard path grew a canonicalizer for prefixes written in a former or +hyphenated spelling and the gateway path never got one, so a local deployment +addressed as "hosted-vllm/..." came out double-prefixed. Collapsing the two +here is what made that one fix rather than two, and it is fixed -- +`tests/data/wire_model_baseline.json` records the single-prefix result. + +So callers ask here rather than building the prefix themselves -- an invariant +test keeps `model_prefix` / `skip_prefixes` readable only by this module and the +two that decompose an id rather than build one. + +The module owns both directions, because they are one contract seen from two +ends. Outbound is `wire_model`. Inbound -- what id to write down when a user +picks a model -- is `stored_model_id`, and identity between two written ids is +`merge_key`. + +Inbound had the same history and a worse symptom. Two implementations decided +what to store, and they disagreed for most providers: picking a model in the +TUI wrote `glm-4.6` while picking the same one in the wizard wrote `zai/glm-4.6`. +Both landed in the same list, so the list held one model twice and removing +either spelling left the other -- deletion reported success and changed nothing. +Comparing by `merge_key` rather than by string is what makes that impossible. +""" + +from __future__ import annotations + +from raven.providers.registry import ( + ProviderSpec, + find_by_model, + normalize_provider_name, + public_model_prefix, + split_model_id, +) + + +def wire_model(model: str, *, spec: ProviderSpec | None = None, gateway: ProviderSpec | None = None) -> str: + """The id this model is sent under, given who is about to send it. + + ``gateway`` is the gateway or local deployment the request goes through, + already detected from the key and address by ``find_gateway``; when it is + set it decides alone, because the prefix that matters is the one naming the + gateway rather than the vendor behind it. + + ``spec`` is the provider whose client is calling. It selects a non-LiteLLM + client's own convention; the LiteLLM path deliberately ignores it and asks + the model id instead, since an id may name a provider other than the + configured one and routing follows the id. + """ + if gateway is not None: + return _through_gateway(model, gateway) + if spec is not None: + if spec.client == "codex": + return _without_own_prefix(model, spec) + if spec.client == "azure": + # The id lands in a URL path as a deployment name, so its provider + # prefix has to come off -- left on, "azure_openai/" became a path + # segment and the request went to a deployment that does not exist. + # Only reached when no `deployment` is configured; that field is the + # proper home for this, and this is the fallback for configs written + # before it existed. + return _without_own_prefix(model, spec) + return _direct(model) + + +def _through_gateway(model: str, gateway: ProviderSpec) -> str: + """Put the gateway's own prefix in front, replacing the vendor's if asked. + + ``model_prefix`` rather than the raw field: a gateway whose name is already + LiteLLM's declares no driver, and reading the field would drop the prefix + entirely -- which is how a gateway's key came to be posted to the vendor + named in the model id. + """ + prefix = gateway.model_prefix + if gateway.strip_model_prefix: + # One leading vendor segment, not everything but the last: a model id + # may itself contain a slash ("openai/gpt-oss-120b" is Groq's own name + # for it), and keeping only the tail truncated the id being served. + _, model = split_model_id(model) + # Canonicalize first, exactly as the direct path does. Comparing the raw + # string instead did not recognize this provider's own name written in + # another of its spellings, so "hosted-vllm/x" -- which is how a stored id + # for a local deployment is written -- came out as + # "hosted_vllm/hosted-vllm/x". The two branches answering one question + # differently is what this module exists to end. + model = _canonical_prefix(model, gateway, prefix) + if prefix and not model.startswith(f"{prefix}/"): + model = f"{prefix}/{model}" + return model + + +def _direct(model: str) -> str: + """Put the routing vendor's prefix in front, for LiteLLM to route on.""" + spec = find_by_model(model) + prefix = spec.model_prefix if spec else "" + if spec and prefix: + model = _canonical_prefix(model, spec, prefix) + if not any(model.startswith(s) for s in (*spec.skip_prefixes, f"{prefix}/")): + model = f"{prefix}/{model}" + return model + + +def _canonical_prefix(model: str, spec: ProviderSpec, canonical: str) -> str: + """Rewrite a prefix written in a former or hyphenated spelling.""" + if "/" not in model: + return model + prefix, remainder = split_model_id(model) + if prefix not in spec.route_names: + return model + return f"{canonical}/{remainder}" + + +def _without_own_prefix(model: str, spec: ProviderSpec) -> str: + """Drop the provider's own name, which its client does not want on the id. + + Both spellings, because the stored id carries the public one + ("openai-codex/") while a config or a command line may have written the + field name ("openai_codex/"). + """ + for prefix in (f"{public_model_prefix(spec)}/", f"{spec.name}/"): + if model.startswith(prefix): + return model[len(prefix) :] + return model + + +# --------------------------------------------------------------------------- +# Inbound: what to write down, and when two written ids are the same model +# --------------------------------------------------------------------------- + + +def stored_model_id(provider: str, model: str) -> str: + """The canonical form to persist for a model chosen under ``provider``. + + Always names its provider. A bare id is claimed by keyword matching instead, + which sends it wherever those rules land rather than to the section the user + just configured: "gpt-5.6-sol" entered under a Codex section matches OpenAI's + keywords and the request leaves for a provider that does not serve it. + + Three shapes arrive here and only one is prefixed blindly: + + * an id already carrying a name this provider answers to is rewritten to the + canonical spelling, so a former name or a hyphenated one does not become a + second entry for the same model; + * an id carrying a prefix this provider declares it accepts is left alone -- + that is what `skip_prefixes` means, and a model reached through a gateway + says so in its own id; + * anything else is bare, and gets this provider's name in front. + """ + from raven.providers.registry import canonical_provider_name, find_by_name, litellm_spelling + + if not model: + return model + + provider = canonical_provider_name(provider) + spec = find_by_name(provider) + # LiteLLM's own spelling for a vendor Raven has no spec for: it is the prefix + # LiteLLM routes on, and the underscored form is rejected outright. + public = public_model_prefix(spec) if spec else litellm_spelling(provider) + if not public: + return model + + head, rest = split_model_id(model) + if head and (head == normalize_provider_name(public) or (spec and head in spec.route_names)): + return f"{public}/{rest}" + if spec and any(model.startswith(skip) for skip in spec.skip_prefixes): + return model + return f"{public}/{model}" + + +def merge_key(provider: str, model: str) -> str: + """Identity of a stored model id, for comparison and de-duplication. + + Two ids name the same model when they name the same provider and the same + vendor id, whatever spelling either was written in. Comparing the strings + instead is what let one model sit in a list twice under two spellings, with + `remove` matching neither the one the user meant nor reporting that it had + not. + + Takes the provider explicitly so an id written before ids carried one still + matches its qualified form. + """ + from raven.providers.registry import canonical_provider_name, find_by_name + + provider = canonical_provider_name(provider) + spec = find_by_name(provider) + head, rest = split_model_id(model or "") + bare = rest if head and (spec and head in spec.route_names or head == normalize_provider_name(provider)) else model + return f"{normalize_provider_name(provider)}::{(bare or '').lower()}" + + +def metadata_candidates(model: str) -> list[str]: + """Which ids to ask LiteLLM's table about, best first. + + A provider reached by region or by subscription is filed in that table under + the vendor's own spelling, and the registry says which + ("minimax-global/MiniMax-M3" -> "minimax/MiniMax-M3"). Asking with the + routing id instead missed every time, and each miss cost a second guess and + the live catalogue fetch behind it. + + Everything else is asked about as it routes, and only then under an + ``openrouter/`` alias: OpenRouter fronts other vendors and prices them under + its own prefix, so the alias covers models LiteLLM lists nowhere else -- but + it answers with OpenRouter's numbers, which are not what a user routing + directly to the vendor pays. Asked alias-first, ``deepseek/deepseek-chat`` + reported a 65,536-token window at $0.14/M where the vendor's own row says + 131,072 at $0.28/M. + """ + from raven.providers.registry import metadata_model_id + + filed_as = metadata_model_id(model) + if filed_as: + return [filed_as] + + if model.startswith("openrouter/"): + return [model] + + return [model, f"openrouter/{model}"] diff --git a/raven/token_wise/cache_optimizer.py b/raven/token_wise/cache_optimizer.py index ae878cdb..9b2546b7 100644 --- a/raven/token_wise/cache_optimizer.py +++ b/raven/token_wise/cache_optimizer.py @@ -33,21 +33,17 @@ from loguru import logger -from raven.providers.registry import find_by_keywords, find_by_model +from raven.providers.prompt_cache import CACHE_CONTROL from raven.token_wise.base import TokenStrategy -_CACHE_CONTROL = {"type": "ephemeral"} +_CACHE_CONTROL = CACHE_CONTROL def _supports_cache_control(model: str) -> bool: - if not model: - return False - # An id routed through a vendor we carry no spec for - # ("bedrock/anthropic.claude-...") resolves to nothing, so fall back to - # keywords: caching is the upstream vendor's capability and survives being - # reached through someone else. - spec = find_by_model(model) or find_by_keywords(model) - return spec is not None and spec.supports_prompt_caching + """Asked of ``providers.prompt_cache`` -- see it for why (wire x family).""" + from raven.providers.prompt_cache import accepts_cache_control + + return accepts_cache_control(model) def _last_index(messages: list[dict[str, Any]], *, role: str) -> int | None: diff --git a/raven/token_wise/pricing.py b/raven/token_wise/pricing.py index ab52119d..abf930f5 100644 --- a/raven/token_wise/pricing.py +++ b/raven/token_wise/pricing.py @@ -1,21 +1,15 @@ -"""Single source of truth for LLM call cost estimation. +"""What a call cost, given what it used. -Used by ``UsageTracker`` and ``BudgetAlerter``. Returning a consistent cost -from one place prevents drift between "what we tracked" and "what we -budgeted". +Used by ``UsageTracker`` and ``BudgetAlerter``. Returning a consistent cost from +one place prevents drift between "what we tracked" and "what we budgeted". -Pricing sources (in order): - 1. ``litellm.cost_per_token`` — covers most public models. Tries the - ``openrouter/`` alias first, then the bare model id. - 2. OpenRouter ``/api/v1/models`` — live per-token prices for any model - LiteLLM lags on, used as a cross-provider catalog (cached 1h in-process). - 3. ``_FALLBACK_PRICING`` — manual rate table for models missing from - both. - 4. ``None`` — model unknown to all. Caller should degrade gracefully. +The rates themselves are a fact about the provider's catalogue, so they come from +``raven.providers.rates``; what lives here is the arithmetic on top of them -- +including Anthropic's ephemeral cache pricing, which is a billing rule rather +than a rate: -Anthropic ephemeral cache pricing is applied on top of the base rate: - cache read → 10% of prompt rate - cache write → 125% of prompt rate (ephemeral 5-min TTL) + cache read -> 10% of prompt rate + cache write -> 125% of prompt rate (ephemeral 5-min TTL) Non-Anthropic providers (no cache support) pass ``cache_read_tokens=0``, ``cache_write_tokens=0`` and the function collapses to the standard formula. @@ -23,442 +17,14 @@ from __future__ import annotations -import pathlib -import threading -import time -from functools import lru_cache - -import httpx from loguru import logger -from raven.token_wise import model_catalog_cache - -# Rate pair: (prompt_cost_per_token, completion_cost_per_token) in USD. -# Keep this table small — it is a fallback for brand-new models that -# LiteLLM hasn't indexed yet. Check LiteLLM first before adding here. -_FALLBACK_PRICING: dict[str, tuple[float, float]] = { - # OpenRouter model pages (snapshot 2026-03) - "z-ai/glm-4.5-air": (0.13e-6, 0.85e-6), # $0.13/$0.85 per 1M -} +from raven.providers.rates import is_plan_billed, token_rates # Track which unknown models we've already warned about so we log once each. _WARNED_UNKNOWN: set[str] = set() -# Live OpenRouter price table, fetched lazily and cached in-process for 1h. -# Maps both the full id (``deepseek/deepseek-v4-pro``) and the bare alias -# (``deepseek-v4-pro``) to OpenRouter's per-token ``pricing`` dict. -_OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" -_OPENROUTER_CACHE_TTL = 3600 -_OPENROUTER_CACHE: dict[str, dict] = {} -_OPENROUTER_CACHE_TIME: float = 0.0 -# Monotonic stamp of the last background warm attempt (0 = never), and how -# long a failed one waits before another is allowed. See -# warm_catalog_in_background. -_WARM_AT: float = 0.0 -_WARM_RETRY_SECONDS = 300.0 - - -def _litellm_price_table() -> dict: - """LiteLLM's static price table, or an empty dict if it cannot be imported.""" - try: - from raven.providers.litellm_setup import import_litellm - - return getattr(import_litellm(), "model_cost", None) or {} - except Exception: - return {} - - -def _table_entry(model: str) -> dict | None: - """The table row keyed exactly by this model id, or None. - - Deliberately no prefix-stripping fallback. It looked like a free replacement - for asking LiteLLM, but the ask does more than key normalization: for an - "openrouter//" candidate it derives OpenRouter's own numbers, - which are in no table row -- and stripping to the direct row answered three - MiniMax models with the direct figure where LiteLLM had reported OpenRouter's. - The table is read here to skip the ask where the ask cannot be made; it does - not replace it. - """ - entry = _litellm_price_table().get(model) - return entry if isinstance(entry, dict) else None - - -@lru_cache(maxsize=1) -def _drivers_dir() -> pathlib.Path | None: - """Where the installed LiteLLM keeps its per-provider drivers.""" - try: - from raven.providers.litellm_setup import import_litellm - - return pathlib.Path(import_litellm().__file__).parent / "llms" - except Exception: - return None - - -def _may_prompt(model: str) -> bool: - """Would handing this model to LiteLLM start an interactive login? - - Three of its drivers ship a device-flow authenticator, and every entry point - that resolves a model reaches it -- ``get_model_info``, ``cost_per_token`` and - ``validate_environment`` alike. With no token file the call prints a device - code to stdout and blocks about a minute per attempt, three attempts per - candidate: one window lookup cost 410 seconds and six codes, because the bare - and the openrouter-prefixed candidate reach the same driver. That is why any - segment counts, not just the first. - - Asked of the installed package rather than a snapshot of it -- a driver is one - that ships ``authenticator.py``. A frozen list would have to be regenerated on - every LiteLLM bump, and a stale one brings the hang back for the vendor it - missed; this cannot go stale. The check is a stat, and the callers have already - paid for the import. - """ - drivers = _drivers_dir() - if drivers is None: - return False - return any((drivers / part / "authenticator.py").exists() for part in model.split("/") if part) - - -def _numeric(entry: dict | None, *fields: str) -> float | None: - """First numeric value among ``fields``, or None. - - The table ships a self-documenting sample row whose numeric-looking fields - hold prose, so the type check is load-bearing rather than defensive. - """ - if not isinstance(entry, dict): - return None - for field in fields: - value = entry.get(field) - if isinstance(value, (int, float)) and value: - return float(value) - return None - - -def is_plan_billed(model: str) -> bool: - """Is this model's provider billed by subscription rather than per token? - - Asked wherever a dollar figure is about to be reported, because on a - subscription there is no per-token figure to report -- not even zero, which - reads as free. - """ - from raven.providers.registry import find_by_model - - spec = find_by_model(model) - return bool(spec and spec.billing == "plan") - - -def _candidates(model: str) -> list[str]: - """Which ids to ask LiteLLM's table about, best first. - - A provider reached by region or by subscription is filed in that table under - the vendor's own spelling, and the registry says which - ("minimax-global/MiniMax-M3" -> "minimax/MiniMax-M3"). Asking with the - routing id instead missed every time, and each miss cost a second guess and - the live catalogue fetch behind it. - - Everything else is asked about as it routes, and only then under an - ``openrouter/`` alias: OpenRouter fronts other vendors and prices them under - its own prefix, so the alias covers models LiteLLM lists nowhere else -- but - it answers with OpenRouter's numbers, which are not what a user routing - directly to the vendor pays. Asked alias-first, ``deepseek/deepseek-chat`` - reported a 65,536-token window at $0.14/M where the vendor's own row says - 131,072 at $0.28/M. - """ - from raven.providers.registry import metadata_model_id - - filed_as = metadata_model_id(model) - if filed_as: - return [filed_as] - - if model.startswith("openrouter/"): - return [model] - - return [model, f"openrouter/{model}"] - - -def _try_litellm_rates(model: str, input_tokens: int, output_tokens: int) -> tuple[float, float] | None: - """Ask LiteLLM for per-token rates. Returns (prompt_rate, completion_rate) or None.""" - try: - from raven.providers.litellm_setup import import_litellm - - litellm = import_litellm() - except Exception: - return None - - candidates = _candidates(model) - - # litellm.cost_per_token expects *at least* 1 non-zero token to compute. - # We pass synthetic tokens to recover the per-token rate. - probe_in = input_tokens if input_tokens else 1 - probe_out = output_tokens if output_tokens else 1 - - for candidate in candidates: - if _may_prompt(candidate): - # Skipped, not read from the table: the rows these families have are - # priced at zero, which this function already treats as unknown, so - # reading them would add a branch that cannot fire. The caller falls - # through to the live OpenRouter catalogue and the manual table, which - # is what any model LiteLLM does not price already does. - continue - try: - prompt_cost, completion_cost = litellm.cost_per_token( - model=candidate, prompt_tokens=probe_in, completion_tokens=probe_out - ) - except Exception: - continue - if prompt_cost is None or completion_cost is None: - continue - if prompt_cost == 0 and completion_cost == 0: - # LiteLLM returns (0, 0) when the model is unknown — treat as miss. - continue - return prompt_cost / probe_in, completion_cost / probe_out - - return None - - -def _fetch_openrouter_models() -> dict[str, dict]: - """Return OpenRouter's model table, fetched live and cached 1h in-process. - - Each entry is ``{"pricing": ..., "context_length": ...}``, double-keyed by - full id and bare alias. On any network failure, returns the stale cache - (or an empty dict) — pricing must never raise into the cost path. - """ - global _OPENROUTER_CACHE, _OPENROUTER_CACHE_TIME - - now = time.time() - if _OPENROUTER_CACHE and (now - _OPENROUTER_CACHE_TIME) < _OPENROUTER_CACHE_TTL: - return _OPENROUTER_CACHE - - # Disk tier: warm-start (or pick up a sibling process's fresher fetch) - # from a fresh on-disk cache without touching the network. - disk = model_catalog_cache.load() - if disk is not None and (now - disk[1]) < _OPENROUTER_CACHE_TTL: - _OPENROUTER_CACHE, _OPENROUTER_CACHE_TIME = disk - return _OPENROUTER_CACHE - - try: - with httpx.Client(timeout=10.0) as client: - resp = client.get(_OPENROUTER_MODELS_URL) - resp.raise_for_status() - data = resp.json() - except Exception as exc: - logger.debug("pricing: OpenRouter models fetch failed ({}), degrading", exc) - if _OPENROUTER_CACHE: - return _OPENROUTER_CACHE - if disk is not None: - _OPENROUTER_CACHE, _OPENROUTER_CACHE_TIME = disk - return _OPENROUTER_CACHE - return {} - - cache: dict[str, dict] = {} - for model in data.get("data", []): - model_id = model.get("id", "") - if not model_id: - continue - arch = model.get("architecture") or {} - mods = arch.get("input_modalities") - entry = { - "pricing": model.get("pricing") or {}, - "context_length": model.get("context_length"), - # What the model accepts as input ("text" / "image" / "audio" / - # "file" / "video"). The catalog is fetched for prices, and it is - # also the only published answer to "can this model see" that - # states itself for every model it lists -- see - # ``capabilities.supports_vision``. - "input_modalities": list(mods) if isinstance(mods, list) and mods else None, - } - cache[model_id] = entry - if "/" in model_id: - cache.setdefault(model_id.split("/", 1)[1], entry) - - _OPENROUTER_CACHE = cache - _OPENROUTER_CACHE_TIME = time.time() - model_catalog_cache.save(cache) - return cache - - -def warm_catalog_in_background() -> None: - """Start filling the catalog off the request path, without blocking a turn. - - The pricing path cannot be relied on to do it. It asks LiteLLM's static - table first and only reaches this catalog when that table *misses*, so for - every model LiteLLM does carry -- which is every model Raven ships a default - for -- the catalog is never fetched and a reader like - :func:`openrouter_input_modalities` has nothing to read, forever. - - Called instead of fetching inline because the fetch is synchronous with a - 10s timeout: on a machine that cannot reach the host, doing it in the turn - would stall the turn. A cold caller therefore degrades until the fetch lands. - - Retried on a cooldown rather than attempted once. An attempt that fails - proves nothing about the next one -- the first turn of a session routinely - runs before a VPN is up or a proxy has authenticated -- and a single latched - attempt would leave the reader answering from an empty catalog for the whole - process. A success needs no cooldown: the filled cache is itself the guard. - """ - global _WARM_AT - - if _OPENROUTER_CACHE: - return - now = time.monotonic() - if _WARM_AT and now - _WARM_AT < _WARM_RETRY_SECONDS: - return - _WARM_AT = now - - # Resolved here rather than inside the thread. A thread body that looks the - # name up on entry can lose a race with whoever patched it -- a test seam - # restored between ``start()`` and the thread's first bytecode would send a - # real request from inside the suite and write the real cache file. - fetch = _fetch_openrouter_models - - def _run() -> None: - try: - fetch() - except Exception as exc: # the fetch degrades internally; a thread must not die loudly - logger.debug("pricing: background catalog warm failed ({})", exc) - - threading.Thread(target=_run, name="raven-model-catalog-warm", daemon=True).start() - - -def _cached_catalog_only() -> dict[str, dict]: - """Whatever catalog is already in hand, at any age, without fetching. - - ``_fetch_openrouter_models`` is synchronous with a one-hour TTL, so calling - it from a request path would hand one turn a stall whenever the hour rolls - over. Prices are why that TTL is short; a model's input modalities are not, - so this reader takes a stale table happily and an absent one as "no answer". - Filling an absent one is :func:`warm_catalog_in_background`'s job. - """ - global _OPENROUTER_CACHE - - if _OPENROUTER_CACHE: - return _OPENROUTER_CACHE - disk = model_catalog_cache.load() - if disk is None: - return {} - # Re-checked after the read, not just before it: ``load()`` touches the - # filesystem and releases the GIL, so a background warm can land in that - # window with both a fresher table and a fresh ``_OPENROUTER_CACHE_TIME``. - # Overwriting it with this stale copy would leave that timestamp vouching for - # the wrong table, and the fetch's TTL check would then skip the refetch. - if _OPENROUTER_CACHE: - return _OPENROUTER_CACHE - # Kept so the next lookup does not re-read and re-parse the file. - # ``_OPENROUTER_CACHE_TIME`` is deliberately left alone: the fetch reads it - # to decide freshness, and this table is of unknown age -- good enough for a - # modality question, not to be mistaken for fresh pricing. - _OPENROUTER_CACHE = disk[0] - return _OPENROUTER_CACHE - - -def openrouter_input_modalities(model: str) -> tuple[str, ...] | None: - """What the catalog says ``model`` accepts as input, or ``None``. - - ``None`` means the catalog has no entry (or one written before this field - was kept), never "text only": this source states itself for every model it - lists, so silence is absence rather than a denial. - - Matched on the full id and then the bare alias, case-folded -- the catalog - spells every id it publishes in lower case, while a routed id need not - (``minimax/MiniMax-M2``). Punctuation is *not* normalized away, and that - restraint is the point: an id that survives only a fuzzier match is an id - this catalog does not actually list, and the only thing a wrong match can do - here is deny vision to a model that has it. ``azure/`` and the local - runtimes take a user-chosen deployment or tag name where every other - provider takes a model id, so ``azure/gpt4`` and ``ollama/phi4`` would join - against ``openai/gpt-4`` and ``microsoft/phi-4`` on a punctuation-stripping - key and lose every picture, silently, on a deployment that may well serve a - vision model. Losing the fuzzy tier costs nothing measurable: on the live - catalog every model it additionally matched either already answers "can see" - (the default when there is no answer at all) or is one of these false - denials. - - Reads only what is already cached -- see :func:`_cached_catalog_only`. - """ - key = model.removeprefix("openrouter/").lower() - table = _cached_catalog_only() - entry = table.get(key) - if entry is None and "/" in key: - entry = table.get(key.split("/", 1)[1]) - if not entry: - return None - mods = entry.get("input_modalities") - return tuple(mods) if isinstance(mods, list) and mods else None - - -def _lookup_openrouter_entry(model: str) -> dict | None: - """Resolve a model to its OpenRouter catalog entry. - - Strips a leading ``openrouter/`` then tries the remaining id and its bare - alias. Used as a cross-provider fallback for any model LiteLLM doesn't map, - so the catalog also covers e.g. a direct ``deepseek/...`` route. - """ - key = model.removeprefix("openrouter/") - table = _fetch_openrouter_models() - entry = table.get(key) - if entry is None and "/" in key: - entry = table.get(key.split("/", 1)[1]) - return entry - - -def _try_openrouter_rates(model: str) -> tuple[float, float] | None: - """Look up live OpenRouter per-token rates. Returns rates or None.""" - entry = _lookup_openrouter_entry(model) - if not entry: - return None - pricing = entry.get("pricing") or {} - try: - return float(pricing["prompt"]), float(pricing["completion"]) - except (KeyError, TypeError, ValueError): - return None - - -def _try_litellm_context_window(model: str) -> int | None: - """LiteLLM's static model metadata — offline, covers most mapped providers.""" - try: - from raven.providers.litellm_setup import import_litellm - - litellm = import_litellm() - except Exception: - return None - - for candidate in _candidates(model): - # The table before the ask: it holds every model the interactive-login - # drivers are asked about in practice, and reading it cannot prompt. - window = _numeric(_table_entry(candidate), "max_input_tokens", "max_tokens") - if window: - return int(window) - if _may_prompt(candidate): - continue - try: - info = litellm.get_model_info(candidate) - except Exception: - continue - window = _numeric(info, "max_input_tokens", "max_tokens") - if window: - return int(window) - return None - - -def resolve_context_window(model: str) -> int | None: - """Return a model's real context window in tokens, or None. - - Sources, in order: LiteLLM's static model metadata (offline, covers every - provider it maps), then OpenRouter's live ``/models`` table - (``context_length``) for any model LiteLLM lags on. Unknown models return - None so the caller keeps its configured default. - """ - window = _try_litellm_context_window(model) - if window: - return window - - entry = _lookup_openrouter_entry(model) - if entry: - try: - length = int(entry.get("context_length") or 0) - except (TypeError, ValueError): - length = 0 - if length: - return length - return None +__all__ = ["estimate_cost_usd", "reset_warning_cache"] def estimate_cost_usd( @@ -476,53 +42,32 @@ def estimate_cost_usd( A plan-billed provider returns None as well. The subscription is the price, so no per-token figure describes this call: LiteLLM files those models at - zero, which the tiers below read as "unknown" and answered with the + zero, which the rate ladder reads as "unknown" and would answer with the pay-as-you-go rate the user is not paying -- $2.50 per million for a Copilot seat. Callers already degrade on None; the tokens are still counted. """ if is_plan_billed(model): return None - rates = _try_litellm_rates(model, input_tokens, output_tokens) + rates = token_rates(model, input_tokens, output_tokens) if rates is None: - rates = _try_openrouter_rates(model) - if rates is None: - key = model.removeprefix("openrouter/") - if key in _FALLBACK_PRICING: - rates = _FALLBACK_PRICING[key] - else: - if model not in _WARNED_UNKNOWN: - logger.warning("pricing: unknown model '{}', cost estimate = None", model) - _WARNED_UNKNOWN.add(model) - return None + if model not in _WARNED_UNKNOWN: + logger.warning("pricing: unknown model '{}', cost estimate = None", model) + _WARNED_UNKNOWN.add(model) + return None prompt_rate, completion_rate = rates - cost = ( + return ( input_tokens * prompt_rate + output_tokens * completion_rate + cache_read_tokens * prompt_rate * 0.1 + cache_write_tokens * prompt_rate * 1.25 ) - return cost def reset_warning_cache() -> None: """Clear the set of models we've already logged an 'unknown' warning for. - Only useful for tests — production code should let warnings land once. + Only useful for tests -- production code should let warnings land once. """ _WARNED_UNKNOWN.clear() - - -def reset_openrouter_cache() -> None: - """Clear the in-process OpenRouter catalog cache. - - Only useful for tests — pair it with the ``model_catalog_cache._CACHE_PATH`` - seam to exercise the disk tiers without touching the real ~/.raven/cache/. - """ - global _OPENROUTER_CACHE, _OPENROUTER_CACHE_TIME, _WARM_AT - _OPENROUTER_CACHE = {} - _OPENROUTER_CACHE_TIME = 0.0 - # Reset too, or a warm attempt from an earlier test leaves this one on a - # cooldown it never asked for. - _WARM_AT = 0.0 diff --git a/raven/token_wise/system_and_tail_cache.py b/raven/token_wise/system_and_tail_cache.py index 14c105f1..37aca100 100644 --- a/raven/token_wise/system_and_tail_cache.py +++ b/raven/token_wise/system_and_tail_cache.py @@ -24,21 +24,17 @@ from loguru import logger -from raven.providers.registry import find_by_keywords, find_by_model +from raven.providers.prompt_cache import CACHE_CONTROL from raven.token_wise.base import TokenStrategy -_CACHE_CONTROL = {"type": "ephemeral"} +_CACHE_CONTROL = CACHE_CONTROL def _supports_cache_control(model: str) -> bool: - if not model: - return False - # An id routed through a vendor we carry no spec for - # ("bedrock/anthropic.claude-...") resolves to nothing, so fall back to - # keywords: caching is the upstream vendor's capability and survives being - # reached through someone else. - spec = find_by_model(model) or find_by_keywords(model) - return spec is not None and spec.supports_prompt_caching + """Asked of ``providers.prompt_cache`` -- see it for why (wire x family).""" + from raven.providers.prompt_cache import accepts_cache_control + + return accepts_cache_control(model) def _apply_cache_marker(msg: dict[str, Any]) -> None: diff --git a/raven/tracing/semconv.py b/raven/tracing/semconv.py index 66259577..a53c2a2b 100644 --- a/raven/tracing/semconv.py +++ b/raven/tracing/semconv.py @@ -131,7 +131,10 @@ def _llm_input_payload( - ``prompt``: the latest user message (the current input to this call), - ``historyMessages``: the prior turns only — everything EXCEPT the system message and that latest user message (so it doesn't duplicate them). - ``messages`` keeps the full raw list as the ground truth of what was sent. + ``messages`` keeps the full raw list as handed to the provider, which is not + what went on the wire: the provider adds or removes prompt-cache breakpoints + on copies (``providers.prompt_cache``) after this is recorded. Neither the + presence nor the absence of ``cache_control`` here says what was sent. """ msgs = messages if isinstance(messages, list) else [] system_prompt = "" diff --git a/raven/tui_rpc/methods/config.py b/raven/tui_rpc/methods/config.py index 15567c17..37fc9875 100644 --- a/raven/tui_rpc/methods/config.py +++ b/raven/tui_rpc/methods/config.py @@ -30,7 +30,9 @@ from typing import TYPE_CHECKING, Any, Callable from raven.cli._helpers import load_runtime_config, make_provider -from raven.providers.registry import find_by_model, find_by_name +from raven.providers import pin +from raven.providers.auth import MissingCredentialsError +from raven.providers.wire import stored_model_id from raven.tui_rpc.errors import ( ConfigFieldReadonlyError, ConfigValidationError, @@ -273,50 +275,6 @@ async def config_set( return {"applied": True, "previous": previous} -def _resolve_bare_model_against_pin(raw_value: str) -> str | None: - """Who serves this bare id, when its own text names nobody? - - A bare id that matches no provider's keywords is what a vendor Raven holds no - spec for looks like, and the picker never produces one -- it always sends the - provider alongside. So this is the hand-typed path, where the only other - evidence is the provider currently pinned. - - Rather than guess, ask whether that provider serves the model: its own curated - list first, then the catalogue. If it does, the pin was right and stays. A local - deployment stays too without asking -- its server names whatever models it - likes, and there is no key to mis-route. - - With no such evidence the pin is not kept: it would send one vendor's key to - another, the mis-routing the prefix rules exist to prevent. Returning None - leaves the caller to say so rather than pick a vendor on the user's behalf. - """ - forced = _get_nested(_load_config(), "agents.defaults.provider") - if not forced or forced == "auto": - return "auto" - - spec = find_by_name(forced) - if spec is not None and spec.is_local: - return forced - - from raven.config.update_providers import get_provider_config - from raven.providers.common_models import common_models_for, litellm_models_for - from raven.providers.registry import split_model_id - - try: - configured = get_provider_config(forced, redact_secrets=True).get("models") or [] - except KeyError: - configured = [] - known = [*configured, *common_models_for(forced), *litellm_models_for(forced)] - for candidate in known: - # Stripping the prefix covers every spelling the sources use: the - # catalogue keys ids the way LiteLLM spells the vendor, a hand-added one - # sits in the provider's list bare. - _, bare = split_model_id(candidate) - if bare == raw_value or candidate == raw_value: - return forced - return None - - def _set_model( params: dict, raw_value: Any, @@ -340,23 +298,23 @@ def _set_model( ) # Bare `/model ` carries no provider; derive it from the model so a # previously-forced provider does not silently mis-route the new model. The - # picker always sends one, so this is the hand-typed path. + # picker always sends one, so this is the hand-typed path. The rule itself is + # `providers.pin`, which `raven provider use` asks too. if new_provider is None: - spec = find_by_model(raw_value) - if spec is not None: - new_provider = spec.name - elif "/" in raw_value: - # A prefixed id whose vendor has no spec of ours ("mistral/..."): - # keeping the previously forced provider would send that provider's - # key to this other vendor, so hand routing back to auto-detection. - new_provider = "auto" - else: - new_provider = _resolve_bare_model_against_pin(raw_value) - if new_provider is None: - raise ConfigValidationError( - f"cannot tell which provider serves {raw_value!r}; qualify it as /{raw_value}", - data={"field": "value", "got": raw_value}, - ) + new_provider = pin.resolve(raw_value, pinned=_get_nested(_load_config(), "agents.defaults.provider") or "") + if new_provider is None: + raise ConfigValidationError( + f"cannot tell which provider serves {raw_value!r}; qualify it as /{raw_value}", + data={"field": "value", "got": raw_value}, + ) + + # Stored the way every other surface stores it -- naming its provider -- so + # the three cannot disagree about what was chosen. A hand-typed bare id used + # to be written raw here while the wizard qualified the same input, which is + # the spelling drift the storage rule exists to end. `auto` names nobody, + # so there is no prefix to add. + if new_provider and new_provider != pin.AUTO: + raw_value = stored_model_id(new_provider, raw_value) session_id = params.get("session_id") if isinstance(session_id, str) and session_id and is_turn_active(session_id): @@ -377,6 +335,15 @@ def _set_model( runtime.agents.defaults.provider = new_provider try: built_provider = make_provider(runtime) + except MissingCredentialsError as exc: + # Carried through as the sentence the user needs. `typer.Exit` + # subclasses RuntimeError, so this used to land in the branch below + # and `str(exc)` was the exit code -- the picker said + # `cannot build provider ... error: "1"`. + raise ModelNotAvailableError( + exc.summary, + data={"model": raw_value, "provider": exc.provider, "remedy": exc.remedy}, + ) from exc except (SystemExit, RuntimeError, ValueError) as exc: raise ModelNotAvailableError( f"cannot build provider for model {raw_value!r}", diff --git a/raven/tui_rpc/methods/model.py b/raven/tui_rpc/methods/model.py index 044eb436..70652a1a 100644 --- a/raven/tui_rpc/methods/model.py +++ b/raven/tui_rpc/methods/model.py @@ -30,6 +30,7 @@ reset_provider, set_provider_fields, ) +from raven.providers.auth import credential_status from raven.providers.common_models import common_models_for, litellm_models_for from raven.providers.registry import ( CRED_ENDPOINT, @@ -39,9 +40,8 @@ credential_kind, find_by_model, find_by_name, - needs_public_model_prefix, - public_model_prefix, ) +from raven.providers.wire import stored_model_id from raven.tui_rpc.errors import ( ConfigValidationError, NotSupportedInV01Error, @@ -83,6 +83,8 @@ def _provider_models(slug: str, *, configured: bool) -> list[str]: # providers have no shortlist at all, which is why the picker used to offer # them nothing; the account last because only one provider can be asked and # asking costs a request (see ``_account_models``). + from raven.providers.wire import merge_key + out: list[str] = [] seen: set[str] = set() chain = ( @@ -92,8 +94,11 @@ def _provider_models(slug: str, *, configured: bool) -> list[str]: *_account_models(slug, configured=configured), ) for candidate in chain: - if candidate not in seen: - seen.add(candidate) + # By identity: a model reaching this list from two sources in two + # spellings used to appear twice in the picker. + key = merge_key(slug, candidate) + if key not in seen: + seen.add(key) out.append(candidate) return out @@ -118,6 +123,51 @@ def _account_models(slug: str, *, configured: bool) -> tuple[str, ...]: return tuple(_stored_spelling(slug, model) for model in account_models()) +def _model_labels(slug: str, models: "list[str]") -> dict[str, dict[str, Any]]: + """Display facts for each offered id, skipping the ones nothing describes. + + What the user wrote under ``model_overlay`` wins: they are describing their + own deployment, and for a model no catalogue carries they are the only + source there is. + """ + from raven.providers.catalog import describe + + overlays = _configured_overlays(slug) + out: dict[str, dict[str, Any]] = {} + for model in models: + row = describe(slug, model, overlay=_overlay_for(overlays, slug, model)) + if not row.described: + continue + entry: dict[str, Any] = {"label": row.label} + if row.description: + entry["description"] = row.description + out[model] = entry + return out + + +def _configured_overlays(slug: str) -> dict[str, Any]: + """This provider's user-written model descriptions, keyed by merge key. + + Keyed by identity rather than by the string the user typed, so an overlay + written against a bare id still matches the qualified id the picker offers. + """ + from raven.config.loader import load_config + from raven.providers.wire import merge_key + + try: + section = load_config().providers.get(slug) + except Exception: + return {} + overlay = getattr(section, "model_overlay", None) or {} + return {merge_key(slug, model): value for model, value in overlay.items()} + + +def _overlay_for(overlays: dict[str, Any], slug: str, model: str) -> Any: + from raven.providers.wire import merge_key + + return overlays.get(merge_key(slug, model)) + + def _build_provider_entry(slug: str, *, current_provider: str | None) -> dict[str, Any]: spec = find_by_name(slug) providers = {p["name"]: p for p in list_providers()} @@ -132,6 +182,11 @@ def _build_provider_entry(slug: str, *, current_provider: str | None) -> dict[st models = _provider_models(slug, configured=configured) return { + # Names and one-liners for the ids above, so the picker shows what a + # model is rather than only what it is called on the wire. Omitted for + # ids no catalogue carries -- a local finetune, or a release newer than + # the bundled snapshot -- and the picker falls back to the id for those. + "model_labels": _model_labels(slug, models), "slug": slug, "name": info.get("display_name") or (spec.label if spec else slug), "authenticated": configured, @@ -213,6 +268,9 @@ async def model_save_key(params: dict) -> dict: data={"slug": parsed.slug}, ) kind = credential_kind(parsed.slug) + # The shape drives which fields to ask for; whether the submission is + # complete is `providers.auth`, the same answer every other gate uses. This + # branch chain was the sixth place deciding that independently. if kind in (CRED_ENDPOINT, CRED_LOCAL) and not parsed.api_base: raise ConfigValidationError( f"{label} requires an api_base", @@ -225,7 +283,8 @@ async def model_save_key(params: dict) -> dict: f"{label} is a local deployment and takes no api_key; send api_base instead", data={"slug": parsed.slug, "field": "api_key"}, ) - if kind != CRED_LOCAL and not parsed.api_key: + submitted = {"api_key": parsed.api_key, "api_base": parsed.api_base} + if not credential_status(parsed.slug, submitted).ok and kind != CRED_LOCAL: raise ConfigValidationError( f"{label} requires an api_key", data={"slug": parsed.slug, "field": "api_key"}, @@ -262,20 +321,14 @@ async def model_disconnect(params: dict) -> dict: def _stored_spelling(slug: str, model: str) -> str: - """The id to store for a model the user typed, for the provider they typed it under. + """The id to store for a model the user typed. See ``providers.wire``. - A bare id is claimed by keyword matching rather than by the provider it was - entered for: "gpt-5.6-sol" resolves to OpenAI, so the request leaves for a - provider that does not serve it. The listed models already carry the prefix, - and a typed one has to end up spelled the same way. + This used to prefix only the three providers whose own client strips the + prefix back off, while the wizard prefixed nearly all of them -- so the same + model picked in the two places was written two different ways into the same + list. """ - spec = find_by_name(slug) - if not needs_public_model_prefix(spec) or not model: - return model - - prefix = public_model_prefix(spec) # type: ignore[arg-type] - - return model if model.startswith(f"{prefix}/") else f"{prefix}/{model.split('/')[-1]}" + return stored_model_id(slug, model) async def model_add_model(params: dict) -> dict: diff --git a/raven/tui_rpc/methods/session.py b/raven/tui_rpc/methods/session.py index f1f10f7a..d86a474a 100644 --- a/raven/tui_rpc/methods/session.py +++ b/raven/tui_rpc/methods/session.py @@ -33,9 +33,9 @@ from raven.cli.update_notice import update_notice from raven.config.loader import load_config +from raven.providers.rates import resolve_context_window from raven.session.export import default_export_path, write_transcript from raven.session.manager import SessionManager, new_chat_id -from raven.token_wise.pricing import resolve_context_window from raven.tui_rpc.errors import TurnInProgressError from raven.tui_rpc.methods import turn as turn_module from raven.tui_rpc.methods.system import _raven_version @@ -113,7 +113,7 @@ def _baseline_usage( banner says so rather than opening at $0.00. Zero here read as free until the first turn replaced it, which is the answer this session will never have. """ - from raven.token_wise.pricing import is_plan_billed + from raven.providers.rates import is_plan_billed context_max = config.agents.defaults.context_window_tokens model = getattr(agent_loop, "model", None) diff --git a/raven/tui_rpc/methods/setup.py b/raven/tui_rpc/methods/setup.py index fc7bfe2b..8d6b0580 100644 --- a/raven/tui_rpc/methods/setup.py +++ b/raven/tui_rpc/methods/setup.py @@ -64,19 +64,38 @@ def _detect_provider_configured(payload: dict) -> bool: if not (isinstance(model, str) and model): return False + # `agents.defaults.provider` used to be waved through on its own, as a + # provider signal from configs predating per-provider sections. It is now + # written on every model change, so that branch would let a pinned name + # stand for credentials nobody has -- the gate would pass with an empty + # config. The name still says which section to ask about; whether it holds + # anything is asked below, like every other provider. provider = defaults.get("provider") - if isinstance(provider, str) and provider and provider != _AUTO_SENTINEL: - if provider in {"minimax_global", "minimax_cn"}: - from raven.providers.minimax_oauth import load_token + if isinstance(provider, str) and provider in {"minimax_global", "minimax_cn"}: + from raven.providers.minimax_oauth import load_token - region = "global" if provider == "minimax_global" else "cn" - return load_token(region) is not None - return True + return load_token("global" if provider == "minimax_global" else "cn") is not None providers = payload.get("providers") if isinstance(providers, dict): - if any(isinstance(v, dict) and v.get("apiKey") for v in providers.values()): - return True + # `providers.auth`, like every other gate. Reading `apiKey` off the raw + # payload made this the seventh rule and it disagreed with the other six + # in both directions -- on the exact two configurations this module's + # rewrite was filed to fix. A Gemini section holding only `apiKeyList` + # parked a working install on the setup panel; Azure with a key and no + # address was waved through into a chat that then could not run. + from raven.config.schema import ProvidersConfig + from raven.providers.auth import credential_status + + try: + sections = ProvidersConfig.model_validate(providers) + except Exception: + sections = None + if sections is not None: + for name in providers: + section = sections.get(name) + if section is not None and credential_status(name, section, include_external=True).ok: + return True from raven.providers.registry import split_model_id diff --git a/raven/tui_rpc/models.py b/raven/tui_rpc/models.py index aa2557ec..7e62d3d9 100644 --- a/raven/tui_rpc/models.py +++ b/raven/tui_rpc/models.py @@ -528,6 +528,18 @@ class SkillUnpinResult(_Strict): # --------------------------------------------------------------------------- +class ModelLabel(_Strict): + """How a model reads to a person, for the ids in ``models``. + + Present only for models a catalogue describes; one released since the + bundled snapshot, or served by a local deployment, has no entry and the + picker shows its id. + """ + + label: str + description: str | None = None + + class ModelOptionProvider(_Strict): """One provider row in the ``/model`` picker.""" @@ -538,6 +550,7 @@ class ModelOptionProvider(_Strict): auth_type: str key_env: str | None = None models: list[str] + model_labels: dict[str, ModelLabel] | None = None total_models: int needs_api_base: bool warning: str diff --git a/scripts/refresh_models_dev_snapshot.py b/scripts/refresh_models_dev_snapshot.py new file mode 100644 index 00000000..bc254ebb --- /dev/null +++ b/scripts/refresh_models_dev_snapshot.py @@ -0,0 +1,259 @@ +"""Regenerate the bundled models.dev snapshot. + +Raven ships a trimmed copy of the models.dev catalogue so a fresh install can +label models without a network round trip, and so tests never depend on one. +This script is how that copy is produced -- editing it by hand would leave no +way to tell what it was trimmed from. + + uv run python scripts/refresh_models_dev_snapshot.py + +Source: the project's own repository rather than its ``/api.json`` endpoint. +Both are models.dev; the repository is the one that can be pinned. api.json is +a rendering of these files served by a small project's app, and an outage there +is the failure this snapshot exists to survive -- so the refresh should not +depend on it either. Pinning also makes a refresh reproducible: the recorded +commit sha says exactly which catalogue a snapshot came from, where "whatever +the site returned that day" said nothing. + +What is kept and why: + +* only the providers Raven can reach -- either LiteLLM maps the vendor (so a + prefixed id routes) or Raven carries a ``ProviderSpec`` for it. The full + catalogue is 181 providers, most of which Raven has no way to talk to, and the + repository rejects additions over 1 MiB; +* only the fields a person reads when choosing a model, plus per-model cost. + Cost is reporting -- it prices a call after the fact and never shapes a + request -- so a stale figure costs an inaccurate total, not a wrong call. + Everything that does shape a request (context windows, capability flags) comes + from LiteLLM's own table, which ships with the dependency. That split is + deliberate: a community-maintained file that goes stale or wrong should cost a + label or a decimal, never a mis-sent request. +""" + +from __future__ import annotations + +import io +import json +import sys +import tarfile +import tomllib +import urllib.request +from pathlib import Path +from typing import Any + +REPO = "anomalyco/models.dev" +REF = "dev" +TARBALL = f"https://codeload.github.com/{REPO}/tar.gz/refs/heads/{REF}" +COMMIT_API = f"https://api.github.com/repos/{REPO}/commits/{REF}" + +SNAPSHOT = Path(__file__).resolve().parents[1] / "raven" / "providers" / "data" / "models_dev.json" + +#: Raven's provider name -> models.dev's name for the same vendor. Only the ones +#: that differ; a matching name needs no entry. Absent vendors (VolcEngine, a +#: local Ollama) and Raven-only sections (custom, hosted_vllm) have no upstream +#: row by nature, not by oversight. +PROVIDER_ALIASES: dict[str, str] = { + "gemini": "google", + "dashscope": "alibaba", + "moonshot": "moonshotai", + "azure_openai": "azure", + "github_copilot": "github-copilot", + "minimax_global": "minimax", + "minimax_cn": "minimax-cn", +} + +#: Model fields carried over. ``name``/``description`` are what a picker renders; +#: ``cost`` prices a finished call. Deliberately absent: ``limit`` (a context +#: window sizes trimming, so it must come from the one table that also prices the +#: request) and the capability flags (``ProviderSpec`` answers those about the +#: wire format, which is a different question than what a model can do). +KEEP = ("name", "description", "cost") + + +def upstream_name(provider: str) -> str: + return PROVIDER_ALIASES.get(provider, provider) + + +def reachable_providers(catalogue: dict[str, dict]) -> set[str]: + """Raven-side names worth carrying labels for. + + Two ways to be reachable and neither contains the other: LiteLLM maps ~130 + vendors Raven has no spec for (a prefixed id routes to them with only a key), + and Raven carries specs for gateways and regional instances LiteLLM has never + heard of (aihubmix, siliconflow, minimax_cn). Taking only the first silently + drops those three -- 123 models -- while every total in the summary still + goes up, which is why ``tests/test_provider_catalog.py::LABELLED_PROVIDERS`` + asserts a labelled provider list rather than a total. + """ + from raven.providers.registry import PROVIDERS + + try: + from raven.providers.litellm_setup import import_litellm + + # ``provider_list`` holds ``LlmProviders`` members, whose ``str()`` is + # "LlmProviders.OPENAI" -- comparing that to a vendor name matches nothing + # and silently keeps the union empty. + known = {getattr(p, "value", str(p)) for p in getattr(import_litellm(), "provider_list", [])} + except Exception: # pragma: no cover - litellm ships with the project + known = set() + + inverse = {v: k for k, v in PROVIDER_ALIASES.items()} + wanted = {spec.name for spec in PROVIDERS} + for upstream in catalogue: + raven = inverse.get(upstream, upstream) + # Either spelling counts: the alias table exists because the two sources + # name the same vendor differently, and LiteLLM sides with either one. + if raven in known or upstream in known: + wanted.add(raven) + return wanted + + +def build(providers: dict[str, dict], *, wanted: set[str]) -> dict: + out: dict[str, dict] = {} + for name in sorted(wanted): + upstream = providers.get(upstream_name(name)) + if not upstream: + continue + models = { + model_id: {k: entry[k] for k in KEEP if k in entry} + for model_id, entry in (upstream.get("models") or {}).items() + } + if models: + out[name] = {"models": models} + return out + + +def read_catalogue(archive: bytes) -> dict[str, dict]: + """Parse the repository's ``providers//`` tree into api.json's shape. + + One ``provider.toml`` names the vendor; everything under ``models/`` is one + row keyed by its path, because the id a vendor publishes can itself contain a + slash -- a gateway files ``moonshotai/Kimi-K2.6`` two directories deep, and + reading only the flat level drops every one of them (siliconflow and + openrouter are entirely nested, so both came back empty). + """ + providers: dict[str, dict] = {} + #: ``models//.toml`` -- the vendor's own definition of a model, + #: shared by every provider that resells it. A provider row states only what + #: differs and points ``base_model`` here for the rest. + canonical: dict[str, dict] = {} + + with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as tar: + for member in tar.getmembers(): + if not member.isfile() or not member.name.endswith(".toml"): + continue + parts = member.name.split("/") + if len(parts) < 4 or parts[1] not in {"providers", "models"}: + continue + handle = tar.extractfile(member) + if handle is None: # pragma: no cover - directories filtered above + continue + try: + data: dict[str, Any] = tomllib.loads(handle.read().decode("utf-8")) + except (tomllib.TOMLDecodeError, UnicodeDecodeError) as exc: + print(f" skipping {member.name}: {exc}", file=sys.stderr) + continue + + if parts[1] == "models": + canonical["/".join(parts[2:]).removesuffix(".toml")] = data + continue + + vendor = providers.setdefault(parts[2], {"models": {}}) + if parts[3] == "provider.toml": + vendor["name"] = data.get("name") or parts[2] + elif parts[3] == "models" and len(parts) > 4: + vendor["models"]["/".join(parts[4:]).removesuffix(".toml")] = data + + resolve_inheritance(providers, canonical) + return providers + + +def resolve_inheritance(providers: dict[str, dict], canonical: dict[str, dict]) -> None: + """Fill in what a row inherits from the model it declares as its base. + + A provider reselling someone else's model states only what differs -- its own + price -- and points ``base_model`` at the vendor's definition for the name and + the description. 2915 of the catalogue's rows are written that way, including + every one of github_copilot's and most of azure's. The published api.json + resolves this before serving; reading the files directly does not, and the + failure is quiet: every row is present and every total looks right, the rows + just have no names. That is why ``tests/test_provider_catalog.py`` asserts a + label per provider and not a count. + """ + resolved: dict[str, dict] = {} + + def entry_for(ref: str) -> dict | None: + # The shared definition first: `base_model = "anthropic/claude-opus-5"` + # names the vendor's model, which is a different file from that vendor's + # own provider row and is the only place some of them exist. + if ref in canonical: + return canonical[ref] + provider, _, model_id = ref.partition("/") + return providers.get(provider, {}).get("models", {}).get(model_id) + + def resolve(ref: str, seen: frozenset[str]) -> dict: + if ref in resolved: + return resolved[ref] + entry = entry_for(ref) + if entry is None: + return {} + base_ref = entry.get("base_model") + # A base can itself derive (alibaba/qwen3.8-max does), so this recurses; + # `seen` stops a cycle from doing it forever. + merged = entry if not base_ref or ref in seen else {**resolve(str(base_ref), seen | {ref}), **entry} + resolved[ref] = merged + return merged + + for provider, vendor in providers.items(): + for model_id in list(vendor["models"]): + ref = f"{provider}/{model_id}" + # A provider row and the shared definition can share a ref; the row + # is the one being resolved, so it seeds `seen` rather than being + # looked up through `entry_for`, which would return the other file. + entry = vendor["models"][model_id] + base_ref = entry.get("base_model") + if base_ref: + entry = {**resolve(str(base_ref), frozenset({ref})), **entry} + vendor["models"][model_id] = entry + + +def _fetch(url: str, *, accept: str = "*/*") -> bytes: + request = urllib.request.Request(url, headers={"User-Agent": "raven-model-catalog-refresh", "Accept": accept}) # noqa: S310 + with urllib.request.urlopen(request, timeout=120) as response: # noqa: S310 - a pinned https URL + return response.read() + + +def main() -> int: + print(f"resolving {REPO}@{REF} ...") + sha = json.loads(_fetch(COMMIT_API, accept="application/vnd.github+json"))["sha"] + print(f"fetching {TARBALL} ({sha[:12]}) ...") + providers = read_catalogue(_fetch(TARBALL)) + print(f" catalogue: {len(providers)} providers") + + wanted = reachable_providers(providers) + snapshot = build(providers, wanted=wanted) + missing = sorted(w for w in wanted if w not in snapshot) + + SNAPSHOT.parent.mkdir(parents=True, exist_ok=True) + # Sorted: the upstream returns models in an unstable order, so an unsorted + # dump makes every refresh a diff of the whole file with no change in it. + # The sha rides along so a snapshot can be traced to the commit it came from. + payload = {"_source": {"repo": REPO, "ref": REF, "sha": sha}, **snapshot} + SNAPSHOT.write_text( + json.dumps(payload, separators=(",", ":"), ensure_ascii=False, sort_keys=True) + "\n", + encoding="utf-8", + ) + + size_mib = SNAPSHOT.stat().st_size / 1048576 + models = sum(len(v["models"]) for v in snapshot.values()) + print(f"wrote {SNAPSHOT.relative_to(Path.cwd())}: {len(snapshot)} providers, {models} models, {size_mib:.3f} MiB") + if missing: + print(f" reachable but not in the catalogue ({len(missing)}): {', '.join(missing)}") + if size_mib > 1: + print("ERROR: over the 1 MiB gate; trim KEEP or PROVIDER_ALIASES", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/conftest.py b/tests/conftest.py index 104f0c88..01547fed 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -107,16 +107,16 @@ def _no_openrouter_network(tmp_path): and mock the transport. The disk cache path is also redirected to a temp file so the real ~/.raven/cache/ is never read or written. """ - from raven.token_wise import model_catalog_cache, pricing + from raven.providers import model_catalog_cache, rates - original_fetch = pricing._fetch_openrouter_models + original_fetch = rates._fetch_openrouter_models original_path = model_catalog_cache._CACHE_PATH - pricing._fetch_openrouter_models = lambda: {} + rates._fetch_openrouter_models = lambda: {} model_catalog_cache._CACHE_PATH = tmp_path / "model-catalog.json" try: yield finally: - pricing._fetch_openrouter_models = original_fetch + rates._fetch_openrouter_models = original_fetch model_catalog_cache._CACHE_PATH = original_path - pricing._OPENROUTER_CACHE.clear() - pricing._OPENROUTER_CACHE_TIME = 0.0 + rates._OPENROUTER_CACHE.clear() + rates._OPENROUTER_CACHE_TIME = 0.0 diff --git a/tests/data/wire_model_baseline.json b/tests/data/wire_model_baseline.json new file mode 100644 index 00000000..469541d5 --- /dev/null +++ b/tests/data/wire_model_baseline.json @@ -0,0 +1,199 @@ +{ + "azure_chat_url": { + "azure-openai/zz-probe-1": "https://example.openai.azure.com/openai/deployments/zz-probe-1/chat/completions?api-version=2024-10-21", + "azure-probe": "https://example.openai.azure.com/openai/deployments/azure-probe/chat/completions?api-version=2024-10-21", + "azure_openai/vendorx/zz-probe-1": "https://example.openai.azure.com/openai/deployments/vendorx/zz-probe-1/chat/completions?api-version=2024-10-21", + "azure_openai/zz-probe-1": "https://example.openai.azure.com/openai/deployments/zz-probe-1/chat/completions?api-version=2024-10-21", + "vendorx/zz-probe-1": "https://example.openai.azure.com/openai/deployments/vendorx/zz-probe-1/chat/completions?api-version=2024-10-21", + "zz-probe-1": "https://example.openai.azure.com/openai/deployments/zz-probe-1/chat/completions?api-version=2024-10-21" + }, + "codex_strip_model_prefix": { + "openai-codex-probe": "openai-codex-probe", + "openai-codex/gpt-5.3-codex": "gpt-5.3-codex", + "openai-codex/zz-probe-1": "zz-probe-1", + "openai_codex/gpt-5.3-codex": "gpt-5.3-codex", + "openai_codex/vendorx/zz-probe-1": "vendorx/zz-probe-1", + "openai_codex/zz-probe-1": "zz-probe-1", + "vendorx/zz-probe-1": "vendorx/zz-probe-1", + "zz-probe-1": "zz-probe-1" + }, + "litellm_resolve_model": { + "aihubmix": { + "aihubmix-probe": "openai/aihubmix-probe", + "aihubmix/vendorx/zz-probe-1": "openai/vendorx/zz-probe-1", + "aihubmix/zz-probe-1": "openai/zz-probe-1", + "openai/zz-probe-1": "openai/zz-probe-1", + "vendorx/zz-probe-1": "openai/zz-probe-1", + "zz-probe-1": "openai/zz-probe-1" + }, + "anthropic": { + "anthropic-probe": "anthropic/anthropic-probe", + "anthropic/claude-sonnet-5": "anthropic/claude-sonnet-5", + "anthropic/vendorx/zz-probe-1": "anthropic/vendorx/zz-probe-1", + "anthropic/zz-probe-1": "anthropic/zz-probe-1", + "vendorx/zz-probe-1": "vendorx/zz-probe-1", + "zz-probe-1": "zz-probe-1" + }, + "azure_openai": { + "azure-openai/zz-probe-1": "azure-openai/zz-probe-1", + "azure-probe": "azure-probe", + "azure_openai/vendorx/zz-probe-1": "azure_openai/vendorx/zz-probe-1", + "azure_openai/zz-probe-1": "azure_openai/zz-probe-1", + "vendorx/zz-probe-1": "vendorx/zz-probe-1", + "zz-probe-1": "zz-probe-1" + }, + "custom": { + "custom/vendorx/zz-probe-1": "openai/vendorx/zz-probe-1", + "custom/zz-probe-1": "openai/zz-probe-1", + "openai/zz-probe-1": "openai/zz-probe-1", + "vendorx/zz-probe-1": "openai/vendorx/zz-probe-1", + "zz-probe-1": "openai/zz-probe-1" + }, + "dashscope": { + "dashscope/qwen-plus": "dashscope/qwen-plus", + "dashscope/vendorx/zz-probe-1": "dashscope/vendorx/zz-probe-1", + "dashscope/zz-probe-1": "dashscope/zz-probe-1", + "openrouter/zz-probe-1": "openrouter/zz-probe-1", + "qwen-probe": "dashscope/qwen-probe", + "vendorx/zz-probe-1": "vendorx/zz-probe-1", + "zz-probe-1": "zz-probe-1" + }, + "deepseek": { + "deepseek-probe": "deepseek/deepseek-probe", + "deepseek/deepseek-v4-flash": "deepseek/deepseek-v4-flash", + "deepseek/vendorx/zz-probe-1": "deepseek/vendorx/zz-probe-1", + "deepseek/zz-probe-1": "deepseek/zz-probe-1", + "vendorx/zz-probe-1": "vendorx/zz-probe-1", + "zz-probe-1": "zz-probe-1" + }, + "gemini": { + "gemini-probe": "gemini/gemini-probe", + "gemini/gemini-2.5-flash": "gemini/gemini-2.5-flash", + "gemini/vendorx/zz-probe-1": "gemini/vendorx/zz-probe-1", + "gemini/zz-probe-1": "gemini/zz-probe-1", + "vendorx/zz-probe-1": "vendorx/zz-probe-1", + "zz-probe-1": "zz-probe-1" + }, + "github_copilot": { + "github-copilot/zz-probe-1": "github_copilot/zz-probe-1", + "github_copilot-probe": "github_copilot/github_copilot-probe", + "github_copilot/gpt-4o": "github_copilot/gpt-4o", + "github_copilot/vendorx/zz-probe-1": "github_copilot/vendorx/zz-probe-1", + "github_copilot/zz-probe-1": "github_copilot/zz-probe-1", + "vendorx/zz-probe-1": "vendorx/zz-probe-1", + "zz-probe-1": "zz-probe-1" + }, + "groq": { + "groq-probe": "groq/groq-probe", + "groq/openai/gpt-oss-120b": "groq/openai/gpt-oss-120b", + "groq/vendorx/zz-probe-1": "groq/vendorx/zz-probe-1", + "groq/zz-probe-1": "groq/zz-probe-1", + "vendorx/zz-probe-1": "vendorx/zz-probe-1", + "zz-probe-1": "zz-probe-1" + }, + "hosted_vllm": { + "hosted-vllm/zz-probe-1": "hosted_vllm/zz-probe-1", + "hosted_vllm/vendorx/zz-probe-1": "hosted_vllm/vendorx/zz-probe-1", + "hosted_vllm/zz-probe-1": "hosted_vllm/zz-probe-1", + "vendorx/zz-probe-1": "hosted_vllm/vendorx/zz-probe-1", + "vllm-probe": "hosted_vllm/vllm-probe", + "vllm/zz-probe-1": "hosted_vllm/zz-probe-1", + "zz-probe-1": "hosted_vllm/zz-probe-1" + }, + "minimax": { + "minimax-probe": "minimax/minimax-probe", + "minimax/vendorx/zz-probe-1": "minimax/vendorx/zz-probe-1", + "minimax/zz-probe-1": "minimax/zz-probe-1", + "openrouter/zz-probe-1": "openrouter/zz-probe-1", + "vendorx/zz-probe-1": "vendorx/zz-probe-1", + "zz-probe-1": "zz-probe-1" + }, + "minimax_cn": { + "anthropic/zz-probe-1": "anthropic/zz-probe-1", + "minimax-cn-probe": "minimax/minimax-cn-probe", + "minimax-cn/MiniMax-M3": "anthropic/MiniMax-M3", + "minimax-cn/zz-probe-1": "anthropic/zz-probe-1", + "minimax_cn/vendorx/zz-probe-1": "anthropic/vendorx/zz-probe-1", + "minimax_cn/zz-probe-1": "anthropic/zz-probe-1", + "vendorx/zz-probe-1": "vendorx/zz-probe-1", + "zz-probe-1": "zz-probe-1" + }, + "minimax_global": { + "anthropic/zz-probe-1": "anthropic/zz-probe-1", + "minimax-global-probe": "minimax/minimax-global-probe", + "minimax-global/MiniMax-M3": "anthropic/MiniMax-M3", + "minimax-global/zz-probe-1": "anthropic/zz-probe-1", + "minimax_global/vendorx/zz-probe-1": "anthropic/vendorx/zz-probe-1", + "minimax_global/zz-probe-1": "anthropic/zz-probe-1", + "vendorx/zz-probe-1": "vendorx/zz-probe-1", + "zz-probe-1": "zz-probe-1" + }, + "moonshot": { + "moonshot-probe": "moonshot/moonshot-probe", + "moonshot/vendorx/zz-probe-1": "moonshot/vendorx/zz-probe-1", + "moonshot/zz-probe-1": "moonshot/zz-probe-1", + "openrouter/zz-probe-1": "openrouter/zz-probe-1", + "vendorx/zz-probe-1": "vendorx/zz-probe-1", + "zz-probe-1": "zz-probe-1" + }, + "ollama_chat": { + "ollama-chat/zz-probe-1": "ollama_chat/zz-probe-1", + "ollama-probe": "ollama_chat/ollama-probe", + "ollama/zz-probe-1": "ollama_chat/zz-probe-1", + "ollama_chat/vendorx/zz-probe-1": "ollama_chat/vendorx/zz-probe-1", + "ollama_chat/zz-probe-1": "ollama_chat/zz-probe-1", + "vendorx/zz-probe-1": "ollama_chat/vendorx/zz-probe-1", + "zz-probe-1": "ollama_chat/zz-probe-1" + }, + "openai": { + "openai-probe": "openai/openai-probe", + "openai/gpt-5.5": "openai/gpt-5.5", + "openai/vendorx/zz-probe-1": "openai/vendorx/zz-probe-1", + "openai/zz-probe-1": "openai/zz-probe-1", + "vendorx/zz-probe-1": "vendorx/zz-probe-1", + "zz-probe-1": "zz-probe-1" + }, + "openai_codex": { + "openai-codex-probe": "openai/openai-codex-probe", + "openai-codex/zz-probe-1": "openai-codex/zz-probe-1", + "openai_codex/vendorx/zz-probe-1": "openai_codex/vendorx/zz-probe-1", + "openai_codex/zz-probe-1": "openai_codex/zz-probe-1", + "vendorx/zz-probe-1": "vendorx/zz-probe-1", + "zz-probe-1": "zz-probe-1" + }, + "openrouter": { + "openrouter-probe": "openrouter/openrouter-probe", + "openrouter/anthropic/claude-sonnet-4-5": "openrouter/anthropic/claude-sonnet-4-5", + "openrouter/vendorx/zz-probe-1": "openrouter/vendorx/zz-probe-1", + "openrouter/zz-probe-1": "openrouter/zz-probe-1", + "vendorx/zz-probe-1": "openrouter/vendorx/zz-probe-1", + "zz-probe-1": "openrouter/zz-probe-1" + }, + "siliconflow": { + "openai/zz-probe-1": "openai/zz-probe-1", + "siliconflow-probe": "openai/siliconflow-probe", + "siliconflow/vendorx/zz-probe-1": "openai/vendorx/zz-probe-1", + "siliconflow/zz-probe-1": "openai/zz-probe-1", + "vendorx/zz-probe-1": "openai/vendorx/zz-probe-1", + "zz-probe-1": "openai/zz-probe-1" + }, + "volcengine": { + "vendorx/zz-probe-1": "volcengine/vendorx/zz-probe-1", + "volcengine-probe": "volcengine/volcengine-probe", + "volcengine/vendorx/zz-probe-1": "volcengine/vendorx/zz-probe-1", + "volcengine/zz-probe-1": "volcengine/zz-probe-1", + "zz-probe-1": "volcengine/zz-probe-1" + }, + "zai": { + "hosted_vllm/zz-probe-1": "hosted_vllm/zz-probe-1", + "openrouter/zz-probe-1": "openrouter/zz-probe-1", + "vendorx/zz-probe-1": "vendorx/zz-probe-1", + "zai/glm-4.6": "zai/glm-4.6", + "zai/vendorx/zz-probe-1": "zai/vendorx/zz-probe-1", + "zai/zz-probe-1": "zai/zz-probe-1", + "zhipu-probe": "zai/zhipu-probe", + "zhipu/zz-probe-1": "zai/zz-probe-1", + "zz-probe-1": "zz-probe-1" + } + } +} diff --git a/tests/integration/test_provider_real_llm.py b/tests/integration/test_provider_real_llm.py new file mode 100644 index 00000000..78ba2cf7 --- /dev/null +++ b/tests/integration/test_provider_real_llm.py @@ -0,0 +1,269 @@ +"""Real requests through the provider stack, because mocks cannot see this layer. + +Every ``acompletion`` in the unit suite is mocked and the wire-format baseline is +generated by the code it checks, so "the id we send is right" is a closed loop: +it asserts the string equals the string that was recorded. A green unit suite +therefore says nothing about the wire, which is what this file is for. + +So this file sends. It is opt-in: without a key every test skips, and it is not +part of the default run (``tests/integration`` is excluded from ``uv run pytest`` +by ``norecursedirs``). Run it deliberately:: + + OPENROUTER_API_KEY=... uv run pytest tests/integration/test_provider_real_llm.py -v + +What it covers is the matrix the mocked tests structurally cannot: + +* the gateway routing shapes -- a prefixed id through the gateway, and a bare + id resolved by the pin; +* the cache-breakpoint rule, as the token counts the vendor actually billed; +* the learned downgrade for an upstream that refuses the field; +* the credential probe, against a live endpoint rather than a mock transport. + +Costs a handful of cheap completions per run, capped at 16 output tokens each. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from raven.providers.base import GenerationSettings +from raven.providers.litellm_provider import LiteLLMProvider + +pytestmark = pytest.mark.asyncio + + +def _config_key(provider: str) -> str: + """A key from the user's own config, so a configured machine needs no env.""" + path = Path.home() / ".raven" / "config.json" + try: + section = json.loads(path.read_text(encoding="utf-8")).get("providers", {}).get(provider) or {} + except Exception: + return "" + return section.get("apiKey") or section.get("api_key") or "" + + +OPENROUTER_KEY = os.environ.get("OPENROUTER_API_KEY") or _config_key("openrouter") + +requires_openrouter = pytest.mark.skipif(not OPENROUTER_KEY, reason="no OpenRouter credential (env or ~/.raven)") + +#: Long enough to clear every vendor's minimum cacheable prefix. +FILLER = "You are a meticulous release engineer. Track every constraint stated below and never restate it. " * 420 + +ANTHROPIC_MODEL = "openrouter/anthropic/claude-fable-5" +NON_ANTHROPIC_MODEL = "openrouter/google/gemini-3.5-flash" +#: OpenRouter routes this one to Amazon Bedrock, whose dialect is `cachePoint`. +REFUSING_MODEL = "openrouter/anthropic/claude-3-haiku" + + +def _provider(model: str, *, provider_name: str = "openrouter") -> LiteLLMProvider: + client = LiteLLMProvider(api_key=OPENROUTER_KEY, default_model=model, provider_name=provider_name) + client.generation = GenerationSettings(temperature=0, max_tokens=16, timeout=120) + return client + + +def _messages(prompt: str = "Reply with the single word: ready.") -> list[dict]: + return [{"role": "system", "content": FILLER}, {"role": "user", "content": prompt}] + + +async def _marked(model: str) -> tuple[list[dict], list[dict] | None]: + """The payload as production sends it -- through the strategy that marks it.""" + from raven.token_wise.cache_optimizer import CacheOptimizer + + messages, tools, _ = await CacheOptimizer().before_llm_call(_messages(), None, model) + return messages, tools + + +def _cached(response) -> int: + usage = response.usage or {} + return int(usage.get("cache_read_input_tokens") or usage.get("cached_tokens") or 0) + + +# --------------------------------------------------------------------------- +# The four routing shapes +# --------------------------------------------------------------------------- + + +@requires_openrouter +async def test_a_gateway_routed_id_reaches_the_upstream_vendor(): + """The prefix names the gateway, the rest names the vendor behind it.""" + response = await _provider(ANTHROPIC_MODEL).chat_with_retry(messages=_messages(), model=ANTHROPIC_MODEL) + + assert response.finish_reason != "error", response.content + assert response.usage.get("prompt_tokens") + + +@requires_openrouter +async def test_a_bare_id_under_a_gateway_still_routes(): + """A stored id can be bare when the provider is pinned; the wire form has to + put the gateway's prefix back on, or the request goes to the vendor direct + with the gateway's key.""" + client = _provider(ANTHROPIC_MODEL) + response = await client.chat_with_retry(messages=_messages(), model="anthropic/claude-fable-5") + + assert response.finish_reason != "error", response.content + + +@requires_openrouter +async def test_the_id_that_goes_out_is_the_one_wire_builds(): + """Asserted against the live call rather than against a recorded string. + + The baseline file is generated by the same code it checks, so it can only say + the behaviour did not change -- never that it is right. A request that comes + back at all is the vendor confirming the id resolved. + """ + from raven.providers.wire import wire_model + + client = _provider(ANTHROPIC_MODEL) + sent = client._resolve_model(ANTHROPIC_MODEL) + + assert sent == wire_model(ANTHROPIC_MODEL, gateway=client._gateway) + response = await client.chat_with_retry(messages=_messages(), model=ANTHROPIC_MODEL) + assert response.finish_reason != "error", response.content + + +# --------------------------------------------------------------------------- +# Cache breakpoints, as the vendor billed them +# --------------------------------------------------------------------------- + + +@requires_openrouter +async def test_an_anthropic_model_still_gets_its_cache_hit(): + """The half of the rule that must not regress. + + Marking is worth keeping for this family: with the field, the second turn + reads nearly the whole prompt back from cache; without it, nothing is + cached. A change that stops marking here is a change to a real bill. + """ + client = _provider(ANTHROPIC_MODEL) + messages, tools = await _marked(ANTHROPIC_MODEL) + + first = await client.chat_with_retry(messages=messages, tools=tools, model=ANTHROPIC_MODEL) + assert first.finish_reason != "error", first.content + second = await client.chat_with_retry(messages=messages, tools=tools, model=ANTHROPIC_MODEL) + + assert _cached(second) > 0, "the marked prefix was not read back from cache" + + +@requires_openrouter +async def test_a_non_anthropic_model_is_billed_the_same_marked_or_not(): + """The half of the rule that was costing money. + + OpenRouter accepts the field on every model it serves and forwards it to + vendors that do not read it: a breakpoint on the last conversation message + billed Gemini for nearly double the prompt tokens the same prompt costs + unmarked. Marked and unmarked must now be indistinguishable, because the + marker no longer fires for this family. + """ + client = _provider(NON_ANTHROPIC_MODEL) + marked_messages, marked_tools = await _marked(NON_ANTHROPIC_MODEL) + + assert "cache_control" not in str(marked_messages), "the strategy marked a family that does not read it" + + marked = await client.chat_with_retry(messages=marked_messages, tools=marked_tools, model=NON_ANTHROPIC_MODEL) + plain = await client.chat_with_retry(messages=_messages(), model=NON_ANTHROPIC_MODEL) + + assert marked.usage.get("prompt_tokens") == plain.usage.get("prompt_tokens"), ( + f"marked={marked.usage} plain={plain.usage} -- the field is still reaching the vendor" + ) + + +@requires_openrouter +async def test_an_upstream_that_refuses_the_field_is_learned_from_once(): + """The failure no table could have predicted. + + Both catalogues correctly report that this model caches; what neither says is + that OpenRouter routes it to Amazon Bedrock, whose dialect is ``cachePoint``. + It answered 400 every turn. The first turn now pays one extra request and + every turn after it is clean. + """ + from raven.providers import prompt_cache + + prompt_cache.reset_suppressions() + client = _provider(REFUSING_MODEL) + + for turn in range(3): + messages, tools = await _marked(REFUSING_MODEL) + response = await client.chat_with_retry(messages=messages, tools=tools, model=REFUSING_MODEL) + assert response.finish_reason != "error", f"turn {turn}: {response.content}" + + assert prompt_cache.is_suppressed(REFUSING_MODEL), "the refusal was not learned" + + +# --------------------------------------------------------------------------- +# The credential probe, against a live endpoint +# --------------------------------------------------------------------------- + + +@requires_openrouter +async def test_the_probe_reports_a_live_gateway_as_usable(): + from raven.config.update_providers import test_provider as probe + + result = probe("openrouter", timeout_s=20) + + assert result["ok"] is True, result + assert result["models_count"], result + + +async def test_a_provider_with_no_catalogue_endpoint_is_not_called_unconfigured(tmp_path): + """Runs without any credential: the point is what the probe *says*, and it + said `not_configured` for seven providers that were configured correctly.""" + from raven.config.update_providers import set_provider_fields + from raven.config.update_providers import test_provider as probe + + config = tmp_path / "config.json" + set_provider_fields("anthropic", {"api_key": "sk-probe"}, config_path=config) + + result = probe("anthropic", timeout_s=10, config_path=config) + + assert result["status"] == "no_probe_endpoint", result + + +# --------------------------------------------------------------------------- +# The streaming path, which is the one the TUI uses +# --------------------------------------------------------------------------- + + +@requires_openrouter +async def test_the_streaming_path_learns_the_refusal_too(): + """The learned downgrade lives in two places because the turn does. + + ``chat_with_retry`` covers the non-streaming callers; the TUI streams, and + that path has no retry ladder at all -- a downgrade learned only behind the + retry ladder never fires for the main interactive surface, and the affected + model goes on answering 400 on every turn there. + """ + from raven.providers import prompt_cache + + prompt_cache.reset_suppressions() + client = _provider(REFUSING_MODEL) + + for turn in range(2): + messages, tools = await _marked(REFUSING_MODEL) + chunks = 0 + async for _ in client.chat_stream(messages=messages, tools=tools, model=REFUSING_MODEL, max_tokens=16): + chunks += 1 + assert chunks, f"turn {turn} streamed nothing" + + assert prompt_cache.is_suppressed(REFUSING_MODEL) + + +@requires_openrouter +async def test_a_stream_that_completes_normally_still_yields_content(): + """The refusal retry reshaped the stream prologue; this is the plain path + it must not disturb.""" + client = _provider(ANTHROPIC_MODEL) + + text = [] + async for delta in client.chat_stream( + messages=[{"role": "user", "content": "Reply with the single word: ready."}], + model=ANTHROPIC_MODEL, + max_tokens=16, + ): + if delta.content: + text.append(delta.content) + + assert "".join(text).strip(), "the stream produced no content" diff --git a/tests/test_agent_loop_usage_sink.py b/tests/test_agent_loop_usage_sink.py index 6bd90c08..66d91359 100644 --- a/tests/test_agent_loop_usage_sink.py +++ b/tests/test_agent_loop_usage_sink.py @@ -16,13 +16,13 @@ import pytest from raven.agent.loop import AgentLoop +from raven.providers import rates from raven.providers.base import LLMProvider, LLMResponse from raven.spine.message import ChatType, Source from raven.spine.turn import Origin, TurnRequest -from raven.token_wise import pricing # The real fetch, captured before conftest's autouse guard stubs it to {}. -_REAL_FETCH = pricing._fetch_openrouter_models +_REAL_FETCH = rates._fetch_openrouter_models class UsageProvider(LLMProvider): @@ -61,9 +61,9 @@ def workspace(): @pytest.fixture(autouse=True) def _reset_openrouter_cache(): - pricing._OPENROUTER_CACHE.clear() + rates._OPENROUTER_CACHE.clear() yield - pricing._OPENROUTER_CACHE.clear() + rates._OPENROUTER_CACHE.clear() def _make_agent(workspace: Path, provider: LLMProvider, model: str, window: int) -> AgentLoop: @@ -120,9 +120,9 @@ def client_factory(*args, **kwargs): kwargs.setdefault("transport", httpx.MockTransport(handler)) return real_client(*args, **kwargs) - monkeypatch.setattr(pricing, "_fetch_openrouter_models", _REAL_FETCH) - monkeypatch.setattr(pricing.httpx, "Client", client_factory) - monkeypatch.setattr(pricing, "_OPENROUTER_CACHE_TIME", 0.0) + monkeypatch.setattr(rates, "_fetch_openrouter_models", _REAL_FETCH) + monkeypatch.setattr(rates.httpx, "Client", client_factory) + monkeypatch.setattr(rates, "_OPENROUTER_CACHE_TIME", 0.0) provider = UsageProvider("openrouter/deepseek/deepseek-v4-pro", 1000, 500) agent = _make_agent( diff --git a/tests/test_azure_openai_provider.py b/tests/test_azure_openai_provider.py index 1adf1bf1..1563360d 100644 --- a/tests/test_azure_openai_provider.py +++ b/tests/test_azure_openai_provider.py @@ -54,3 +54,33 @@ async def test_chat_wall_clock_cap_returns_classified_error(monkeypatch: pytest. assert resp.error_classification is not None assert resp.error_classification.category == "network" assert resp.error_classification.retryable is True + + +def test_a_configured_deployment_decides_the_url_path() -> None: + """The deployment is a connection parameter, not part of the model id. + + It used to be read off the model id, which forced Azure's ids to be spelled + without the prefix every other provider carries -- a connection detail + dictating the shape of a stored model id. + """ + provider = AzureOpenAIProvider( + api_key="k", + api_base="https://x.openai.azure.com", + default_model="azure_openai/gpt-4o", + deployment="my-prod-deployment", + ) + url = provider._build_chat_url("azure_openai/gpt-4o") + assert "/deployments/my-prod-deployment/chat/completions" in url + assert "azure_openai" not in url + + +def test_without_a_deployment_the_model_id_still_names_it() -> None: + """Configs written before the field exists must keep working unchanged.""" + provider = AzureOpenAIProvider(api_key="k", api_base="https://x.openai.azure.com") + assert "/deployments/my-deployment/chat/completions" in provider._build_chat_url("my-deployment") + + +def test_the_api_version_comes_from_config_rather_than_the_client() -> None: + """A tenant on another version had no way to say so while it was hardcoded.""" + provider = AzureOpenAIProvider(api_key="k", api_base="https://x.openai.azure.com", api_version="2025-01-01") + assert provider._build_chat_url("d").endswith("?api-version=2025-01-01") diff --git a/tests/test_bedrock_stub.py b/tests/test_bedrock_stub.py index 6e8b9b56..5952a4ce 100644 --- a/tests/test_bedrock_stub.py +++ b/tests/test_bedrock_stub.py @@ -47,10 +47,35 @@ def test_no_bedrock_provider_in_registry(): assert find_by_model("bedrock/amazon.titan-text-express-v1") is None -def test_only_bedrock_touchpoint_is_the_helpers_key_gate_bypass(): +def test_the_key_gate_tolerates_bedrock_without_asking_for_a_key(): + """Bedrock is admitted, and the reason is declared rather than string-matched. + + The gate used to let it through on ``model.startswith("bedrock/")`` written + into the CLI helper. The tolerance is real -- AWS credentials come from the + environment, so demanding an ``api_key`` would demand something the user does + not have in that form -- but stating it as a prefix test put a fact about how + a vendor authenticates inside a startup check, where nothing else could see + it. It is now one entry in ``providers.auth``, alongside the other shapes. + + Still a stub, not a backend: this says the gate does not stand in the way, + not that Raven routes Bedrock traffic (see the registry test above). + """ + import typer + + from raven.config.schema import Config + from raven.providers.auth import KIND_AMBIENT, credential_status + + assert credential_status("bedrock", None).kind == KIND_AMBIENT + + config = Config.model_validate( + {"providers": {}, "agents": {"defaults": {"model": "bedrock/amazon.titan-text-express-v1"}}} + ) + try: + helpers_mod.check_provider_credentials(config) + except typer.Exit: # pragma: no cover - the failure message is the point + raise AssertionError("the key gate rejected a bedrock model id") from None + source = Path(helpers_mod.__file__).read_text(encoding="utf-8") - # The sole bedrock reference: the key-gate bypass in make_provider. - assert source.count("bedrock") == 1 - assert 'model.startswith("bedrock/")' in source + assert "bedrock" not in source, "the prefix test belongs in providers.auth, not the startup gate" # No Bedrock Converse backend has been wired in. assert "converse" not in source diff --git a/tests/test_cli_helpers.py b/tests/test_cli_helpers.py index 79ca052d..728e125e 100644 --- a/tests/test_cli_helpers.py +++ b/tests/test_cli_helpers.py @@ -123,14 +123,19 @@ def _write_config(tmp_path: Path, *, api_key: str | None) -> Path: return p -def test_check_provider_credentials_exits_when_no_key(tmp_path: Path) -> None: - import typer - +def test_check_provider_credentials_raises_when_no_key(tmp_path: Path) -> None: + """Raises rather than printing and exiting: three entry points ask this, and + only one of them is a terminal. The sentence travels with the exception so + each renders it in its own idiom.""" from raven.config.loader import load_config + from raven.providers.auth import MissingCredentialsError - with pytest.raises(typer.Exit): + with pytest.raises(MissingCredentialsError) as excinfo: _helpers.check_provider_credentials(load_config(_write_config(tmp_path, api_key=None))) + assert "API key" in excinfo.value.summary + assert excinfo.value.provider + def test_check_provider_credentials_passes_with_key(tmp_path: Path) -> None: from raven.config.loader import load_config @@ -201,26 +206,49 @@ def _config_for(tmp_path: Path, provider: str, model: str, section: dict) -> Pat return p -def test_the_credential_check_accepts_an_oauth_provider_with_no_key(tmp_path: Path) -> None: - """Its docstring promises to stay in sync with ``make_provider``, and the version - that compared provider names drifted the moment the factory stopped doing that. - Asserted through behaviour: a source scan passed while the branch was dead. +def test_an_empty_config_section_is_not_a_rejection_for_an_oauth_provider(monkeypatch, tmp_path: Path) -> None: + """A signed-in OAuth provider passes with nothing in its config section. + + Its credential is a token file, so an empty section is the normal shape and + must not read as "no API key". Asserted through behaviour: a source scan + passed while the branch was dead. """ from raven.config.loader import load_config + monkeypatch.setattr("raven.providers.chatgpt_token.stored_credentials", lambda: {"access_token": "t"}) cfg = _config_for(tmp_path, "openai_codex", "openai-codex/gpt-5.6-sol", {}) _helpers.check_provider_credentials(load_config(cfg)) # no raise: the token is not in config -def test_the_credential_check_wants_both_halves_of_an_azure_endpoint(tmp_path: Path) -> None: - import typer +def test_an_oauth_provider_that_was_never_signed_in_is_told_to_sign_in(monkeypatch, tmp_path: Path, capsys) -> None: + """Missing credentials name the fix, rather than surfacing at the first call. + + This gate used to return early for Codex without checking anything, so an + agent configured for it started and then failed on the first request with + whatever the backend said about an absent token. + """ + from raven.config.loader import load_config + from raven.providers.auth import MissingCredentialsError + monkeypatch.setattr("raven.providers.chatgpt_token.stored_credentials", lambda: None) + cfg = _config_for(tmp_path, "openai_codex", "openai-codex/gpt-5.6-sol", {}) + + with pytest.raises(MissingCredentialsError) as excinfo: + _helpers.check_provider_credentials(load_config(cfg)) + + # Carried on the exception rather than printed: the TUI reaches this same + # check and a stdout line there goes to a log nobody is reading. + assert "raven provider login openai-codex" in excinfo.value.summary + + +def test_the_credential_check_wants_both_halves_of_an_azure_endpoint(tmp_path: Path) -> None: from raven.config.loader import load_config + from raven.providers.auth import MissingCredentialsError # A key without an address is the half Azure cannot work with, and the generic # "no API key" message would not say which half is missing. cfg = _config_for(tmp_path, "azure_openai", "my-deployment", {"apiKey": "az-key"}) - with pytest.raises(typer.Exit): + with pytest.raises(MissingCredentialsError): _helpers.check_provider_credentials(load_config(cfg)) diff --git a/tests/test_cli_onboard_commands.py b/tests/test_cli_onboard_commands.py index b1ef74f1..4fb005d2 100644 --- a/tests/test_cli_onboard_commands.py +++ b/tests/test_cli_onboard_commands.py @@ -1939,11 +1939,13 @@ def test_configure_existing_model_happy_path_persists_and_returns_true( ) monkeypatch.setattr(onboard_commands, "_pick_model", lambda provider, spec, **_: "minimax-global/MiniMax-M3") persisted: list[str] = [] - monkeypatch.setattr(onboard_commands, "_persist_default_model", lambda m: persisted.append(m)) + monkeypatch.setattr(onboard_commands, "_persist_default_model", lambda m, provider: persisted.append((m, provider))) monkeypatch.setattr(onboard_commands, "_run_test_probe", lambda *a, **k: "ok") assert onboard_commands._configure_existing_provider_model(non_interactive=False) is True - assert persisted == ["minimax-global/MiniMax-M3"] + # The pin travels with the model: writing one without the other leaves the + # wizard's own choice routed to whatever was pinned before. + assert persisted == [("minimax-global/MiniMax-M3", "minimax_global")] def test_configure_existing_model_verify_failure_returns_false_without_persist( @@ -1953,7 +1955,7 @@ def test_configure_existing_model_verify_failure_returns_false_without_persist( _patch_single_provider_pick(monkeypatch, "openai") monkeypatch.setattr(onboard_commands, "_verify_provider", lambda *a, **k: (False, "invalid_key", None)) persisted: list[str] = [] - monkeypatch.setattr(onboard_commands, "_persist_default_model", lambda m: persisted.append(m)) + monkeypatch.setattr(onboard_commands, "_persist_default_model", lambda m, provider: persisted.append((m, provider))) assert onboard_commands._configure_existing_provider_model(non_interactive=False) is False assert persisted == [] @@ -1964,7 +1966,7 @@ def test_configure_existing_model_reauth_delegates_to_oauth_login(monkeypatch: p _patch_single_provider_pick(monkeypatch, "minimax_global") monkeypatch.setattr(onboard_commands, "_verify_provider", lambda *a, **k: (True, "valid", [])) monkeypatch.setattr(onboard_commands, "_pick_model", lambda provider, spec, **_: "minimax-global/MiniMax-M3") - monkeypatch.setattr(onboard_commands, "_persist_default_model", lambda m: None) + monkeypatch.setattr(onboard_commands, "_persist_default_model", lambda m, provider: None) monkeypatch.setattr(onboard_commands, "_run_test_probe", lambda *a, **k: "reauth") login_calls: list[str] = [] monkeypatch.setattr(onboard_commands, "_run_oauth_login", lambda p: login_calls.append(p) or True) @@ -2610,9 +2612,13 @@ def test_the_wizard_offers_every_provider_the_registry_carries() -> None: from raven.cli.onboard_commands import _CURATED_PROVIDERS from raven.providers.registry import PROVIDERS - offered = {entry["name"] for entry in _CURATED_PROVIDERS} + names = [entry["name"] for entry in _CURATED_PROVIDERS] + offered = set(names) registered = {spec.name for spec in PROVIDERS} assert registered - offered == set(), f"registry providers missing from the wizard: {sorted(registered - offered)}" + # Once each, on top of the two directions already asserted: nothing stopped + # one provider appearing twice under two labels. + assert len(names) == len(offered), f"offered twice: {sorted({n for n in names if names.count(n) > 1})}" assert offered - registered == set(), f"wizard offers providers with no spec: {sorted(offered - registered)}" @@ -2788,7 +2794,7 @@ def test_resolve_model_with_test_runs_for_a_provider_with_no_spec(monkeypatch, t "_verify_provider", lambda provider, skip_test=False: (True, "valid", ["mistral-large-latest"]), ) - monkeypatch.setattr(onboard_commands, "_persist_default_model", lambda model: None) + monkeypatch.setattr(onboard_commands, "_persist_default_model", lambda model, provider: None) chosen = onboard_commands._resolve_model_with_test( "mistral", @@ -3028,7 +3034,7 @@ def ask(self): onboard_commands, "_verify_provider", lambda provider: (True, "valid", ["mistral-large-latest"]) ) monkeypatch.setattr(onboard_commands, "_pick_model", lambda provider, spec, **_: f"{provider}/probe") - monkeypatch.setattr(onboard_commands, "_persist_default_model", lambda model: None) + monkeypatch.setattr(onboard_commands, "_persist_default_model", lambda model, provider: None) # Reaching this without an AttributeError is the second half of the fix: the # probe is told whether the provider is OAuth, read off a spec that is None. monkeypatch.setattr(onboard_commands, "_run_test_probe", lambda provider, **kw: "ok") @@ -3886,7 +3892,7 @@ def test_removing_a_spec_less_provider_warns_when_it_serves_the_default_model( from raven.config.update_providers import set_provider_fields set_provider_fields("mistral", {"api_key": "sk-mistral"}) - onboard_commands._persist_default_model("mistral/mistral-large-latest") + onboard_commands._persist_default_model("mistral/mistral-large-latest", "mistral") asked: list[str] = [] @@ -4288,3 +4294,45 @@ def _never(**_): result = runner.invoke(app, ["tui"]) assert "openai-codex/gpt-5.6-sol" in result.output, "the notice did not name the model to fix" + + +# --------------------------------------------------------------------------- +# The wizard's vendor list against the registry +# --------------------------------------------------------------------------- + + +def test_each_provider_sits_in_the_group_its_credentials_put_it_in() -> None: + """A vendor filed under the wrong heading is asked for the wrong thing. + + The group decides which prompt the wizard runs -- a key, a sign-in, or an + address -- so it has to follow the declared connection shape rather than + where a hand edit happened to put the row. + """ + from raven.providers.auth import KIND_API_KEY, KIND_DEVICE_FLOW, KIND_NONE, credential_status + + # Every kind maps to exactly one group. Defaulting the unlisted kinds to + # "whatever group this row is already in" made the check tautological for + # them: a key-based provider filed under "oauth" compared "oauth" against + # "oauth" and passed, so only one of the two directions was ever tested. + # No entry for `ambient`: no provider the wizard offers declares it (Bedrock, + # the only one, has no spec and is not offered). Mapping it anyway would be + # guessing at a group for a row that cannot appear -- and the assertion below + # turns its arrival into an explicit decision rather than a silent default. + group_for_kind = { + KIND_DEVICE_FLOW: "oauth", + KIND_NONE: "local", + KIND_API_KEY: "api_key", + } + misfiled = [] + for group in onboard_commands._CURATED_GROUPS: + if group["kind"] == "fallback": + continue # not a provider group: the vendor search and the generic endpoint + for entry in group["providers"]: + if entry["name"] == onboard_commands._PICK_LITELLM_VENDOR: + continue + kind = credential_status(entry["name"], None).kind + want = group_for_kind.get(kind) + assert want, f"{entry['name']}: credential kind {kind!r} maps to no group" + if group["kind"] != want: + misfiled.append(f"{entry['name']}: filed under {group['kind']!r}, credentials say {want!r}") + assert not misfiled, "; ".join(misfiled) diff --git a/tests/test_cli_provider_commands.py b/tests/test_cli_provider_commands.py index afdf5208..beb44d86 100644 --- a/tests/test_cli_provider_commands.py +++ b/tests/test_cli_provider_commands.py @@ -377,7 +377,6 @@ def test_show_lists_all_flags(tmp_config: Path) -> None: def test_show_gemini_includes_extra_flags(tmp_config: Path) -> None: r = runner.invoke(app, ["provider", "show", "gemini"]) assert r.exit_code == 0 - assert "--vertex" in r.stdout assert "--api-key-list" in r.stdout @@ -465,12 +464,27 @@ def test_set_with_equals_form(tmp_config: Path) -> None: assert data["providers"]["openrouter"]["apiKey"] == "sk-equals" -def test_set_with_no_vertex_bool_negative(tmp_config: Path) -> None: - runner.invoke(app, ["provider", "set", "gemini", "--vertex", "true"]) - r = runner.invoke(app, ["provider", "set", "gemini", "--no-vertex"]) - assert r.exit_code == 0, r.output - data = json.loads(tmp_config.read_text(encoding="utf-8")) - assert data["providers"]["gemini"]["vertex"] is False +def test_the_three_bool_flag_forms_are_parsed(monkeypatch) -> None: + """Tested against the parser rather than a provider, because none declares a bool. + + Gemini's ``vertex`` was the only one and it has been removed. The parser + keeps the forms -- it mirrors ``_parse_channel_flags``, where bool fields are + common -- so this exercises them directly instead of asserting through a + field that would have to be invented to keep the test alive. + """ + from raven.cli import provider_commands + + # Patched where it is looked up: the parser imports it inside the function, + # so the name on `provider_commands` is never the one that gets called. + monkeypatch.setattr( + "raven.config.update_providers.provider_field_specs", + lambda name: {"dry_run": {"type": "bool", "default": False, "is_secret": False, "description": ""}}, + ) + parse = provider_commands._parse_provider_flags + # A written value comes back as written; the schema coerces it later. + assert parse(["--dry-run", "true"], "gemini") == {"dry_run": "true"} + assert parse(["--dry-run"], "gemini") == {"dry_run": True} + assert parse(["--no-dry-run"], "gemini") == {"dry_run": False} def test_reset_without_yes_aborts_on_no(tmp_config: Path) -> None: @@ -675,3 +689,167 @@ def test_resetting_the_provider_behind_the_default_model_names_the_way_back( flat = " ".join(r.stdout.split()) assert "no longer works" in flat, flat assert expected in flat, flat + + +# --------------------------------------------------------------------------- +# `provider use`: the third surface that can change the model +# --------------------------------------------------------------------------- + + +def test_use_sets_the_default_model_in_the_shared_spelling(tmp_config: Path) -> None: + """The CLI writes what the wizard and the picker write. + + Three surfaces choose models and only two could; the third had to re-run the + whole wizard. Now that all three write, they have to write the same string -- + that is the contract the storage step established. + """ + from raven.providers.wire import stored_model_id + + r = runner.invoke(app, ["provider", "use", "claude-sonnet-5", "--provider", "anthropic"]) + assert r.exit_code == 0, r.output + + data = json.loads(tmp_config.read_text(encoding="utf-8")) + stored = data["agents"]["defaults"]["model"] + assert stored == stored_model_id("anthropic", "claude-sonnet-5") == "anthropic/claude-sonnet-5" + + +def test_use_infers_the_provider_from_a_qualified_id(tmp_config: Path) -> None: + r = runner.invoke(app, ["provider", "use", "deepseek/deepseek-chat"]) + assert r.exit_code == 0, r.output + data = json.loads(tmp_config.read_text(encoding="utf-8")) + assert data["agents"]["defaults"]["model"] == "deepseek/deepseek-chat" + + +def test_use_warns_but_does_not_refuse_when_the_provider_has_no_credentials(tmp_config: Path) -> None: + """Picking a model before configuring its provider is a normal order. + + Refusing would force the two steps into one sequence; the startup gate says + the same thing again if the key is still missing by the time it matters. + """ + r = runner.invoke(app, ["provider", "use", "deepseek/deepseek-chat"]) + assert r.exit_code == 0, r.output + assert "API key" in r.output + + +def test_use_accepts_a_bare_id_when_nothing_is_pinned(tmp_config: Path) -> None: + """With no pin there is no key to mis-route, so auto-detection is the answer. + + This is where the CLI and the picker used to disagree: the CLI refused any id + that named nobody, the picker stored it against ``auto``. Refusing is for the + case where a pin exists and does *not* serve the model -- there, keeping it + would send one vendor's key to another and dropping it silently would discard + something the user set on purpose, so the only honest move is to ask. + """ + r = runner.invoke(app, ["provider", "use", "some-unqualified-model"]) + + assert r.exit_code == 0, r.output + defaults = json.loads(tmp_config.read_text(encoding="utf-8"))["agents"]["defaults"] + assert defaults["provider"] == "auto" + assert defaults["model"] == "some-unqualified-model" + + +def test_use_moves_the_pin_instead_of_reporting_that_it_is_stuck(tmp_config: Path) -> None: + """A write that changes nothing must not report success and stop there. + + `agents.defaults.provider` overrides what a model id names, so `provider use` + wrote the model, printed a tick, and requests kept going to the pinned + provider. It used to say so and tell the user to set the field to 'auto' -- + advice no command could follow, because none wrote that field. It writes it + now, by the same rule the picker uses. + """ + tmp_config.write_text(json.dumps({"agents": {"defaults": {"provider": "openai"}}}), encoding="utf-8") + # Configured, because an unconfigured vendor is deliberately left on `auto` + # so a gateway can serve it -- see test_use_does_not_pin_a_vendor_that_has_no_configuration. + runner.invoke(app, ["provider", "set", "anthropic", "--api-key", "sk-ant"]) + + r = runner.invoke(app, ["provider", "use", "anthropic/claude-sonnet-5"]) + + assert r.exit_code == 0, r.output + defaults = json.loads(tmp_config.read_text(encoding="utf-8"))["agents"]["defaults"] + assert defaults["provider"] == "anthropic" + assert defaults["model"] == "anthropic/claude-sonnet-5" + assert "pinned" not in r.output, "the note is about a state that can no longer happen" + + +def test_use_hands_routing_back_to_auto_for_a_vendor_with_no_spec(tmp_config: Path) -> None: + """Keeping the old pin would send its key to a vendor it does not belong to.""" + tmp_config.write_text(json.dumps({"agents": {"defaults": {"provider": "openai"}}}), encoding="utf-8") + + r = runner.invoke(app, ["provider", "use", "mistral/mistral-large"]) + + assert r.exit_code == 0, r.output + defaults = json.loads(tmp_config.read_text(encoding="utf-8"))["agents"]["defaults"] + assert defaults["provider"] == "auto" + assert defaults["model"] == "mistral/mistral-large" + + +def test_use_keeps_a_pin_that_serves_the_bare_id(tmp_config: Path) -> None: + """A bare id names nobody, so the pin is the only evidence -- and it is kept + only when the pinned provider actually serves the model.""" + tmp_config.write_text(json.dumps({"agents": {"defaults": {"provider": "deepseek"}}}), encoding="utf-8") + runner.invoke(app, ["provider", "set", "deepseek", "--api-key", "sk-ds"]) + + r = runner.invoke(app, ["provider", "use", "deepseek-chat"]) + + assert r.exit_code == 0, r.output + defaults = json.loads(tmp_config.read_text(encoding="utf-8"))["agents"]["defaults"] + assert defaults["provider"] == "deepseek" + assert defaults["model"] == "deepseek/deepseek-chat" + + +def test_use_leaves_the_config_alone_when_it_cannot_tell(tmp_config: Path) -> None: + """Refusing is the point: writing a guess would route one vendor's key to + another, which is what the prefix rules exist to prevent.""" + before = json.dumps({"agents": {"defaults": {"provider": "deepseek", "model": "deepseek/deepseek-chat"}}}) + tmp_config.write_text(before, encoding="utf-8") + + r = runner.invoke(app, ["provider", "use", "some-model-nobody-serves"]) + + assert r.exit_code == 1 + assert json.loads(tmp_config.read_text(encoding="utf-8")) == json.loads(before) + + +def test_use_says_so_when_an_azure_deployment_overrides_the_model_id(tmp_config: Path) -> None: + """Azure's deployment decides the deployment; the model id then does nothing.""" + runner.invoke(app, ["provider", "set", "azure-openai", "--api-key", "k", "--api-base", "https://x/"]) + runner.invoke(app, ["provider", "set", "azure-openai", "--deployment", "prod-gpt4"]) + + r = runner.invoke(app, ["provider", "use", "azure-openai/some-model"]) + assert r.exit_code == 0, r.output + assert "deployment" in r.output and "prod-gpt4" in r.output + + +def test_use_does_not_pin_a_vendor_that_has_no_configuration(tmp_config: Path) -> None: + """Pinning an unconfigured vendor makes the install unroutable. + + A pin is consulted before anything else and is answered with that vendor's + section whether or not it holds credentials, so pinning an unconfigured one + fails every request on a missing key -- never reaching the fallback written + for exactly this shape, a gateway serving a model whose id names the vendor + behind it. An OpenRouter-only install that ran `provider use anthropic/...` + could then reach nothing at all. + """ + from raven.config.loader import load_config + + runner.invoke(app, ["provider", "set", "openrouter", "--api-key", "sk-or"]) + + r = runner.invoke(app, ["provider", "use", "anthropic/claude-sonnet-4-5"]) + assert r.exit_code == 0, r.output + + defaults = json.loads(tmp_config.read_text(encoding="utf-8"))["agents"]["defaults"] + assert defaults["provider"] == "auto", "an unconfigured vendor must not be pinned" + + section, name = load_config()._match_provider(defaults["model"]) + assert (section, name) != (None, None), "the config was left unable to route" + assert name == "openrouter" + assert "/" in r.output, "the advice must not be 'buy a key from that vendor'" + + +def test_use_still_pins_a_vendor_that_is_configured(tmp_config: Path) -> None: + """The fallback is for the unconfigured case only -- a vendor the user has + set up is still named outright, so its own key is the one used.""" + runner.invoke(app, ["provider", "set", "anthropic", "--api-key", "sk-ant"]) + + r = runner.invoke(app, ["provider", "use", "anthropic/claude-sonnet-4-5"]) + assert r.exit_code == 0, r.output + assert json.loads(tmp_config.read_text(encoding="utf-8"))["agents"]["defaults"]["provider"] == "anthropic" diff --git a/tests/test_config_update_providers.py b/tests/test_config_update_providers.py index c6c594bd..239d64ee 100644 --- a/tests/test_config_update_providers.py +++ b/tests/test_config_update_providers.py @@ -22,6 +22,7 @@ set_provider_fields, ) from raven.config.update_providers import test_provider as probe_provider +from raven.providers.registry import PROVIDERS as _PROVIDERS @pytest.fixture @@ -86,13 +87,12 @@ def test_set_complex_provider_azure(cfg_path: Path) -> None: def test_set_gemini_extra_fields(cfg_path: Path) -> None: set_provider_fields( "gemini", - {"api_key": "g-key", "vertex": "true", "api_key_list": "k1,k2,k3"}, + {"api_key": "g-key", "api_key_list": "k1,k2,k3"}, config_path=cfg_path, ) section = _read(cfg_path)["providers"]["gemini"] assert section["apiKey"] == "g-key" - assert section["vertex"] is True assert section["apiKeyList"] == ["k1", "k2", "k3"] @@ -882,3 +882,177 @@ def cannot_produce_a_token(timeout=5.0, strict=False): assert result["status"] == "oauth_token_missing", result assert "could not renew" in result["error"] + + +def test_a_model_stored_one_way_is_removed_by_the_other(cfg_path: Path) -> None: + """Deletion matches the model, not the spelling. + + The two write paths disagreed for most providers, so a list could hold one + model as both `glm-4.6` and `zai/glm-4.6`. Removing either string left the + other behind, and the call reported success. + """ + from raven.config.update_providers import add_provider_model, remove_provider_model + + add_provider_model("zai", "glm-4.6", config_path=cfg_path) + remaining = remove_provider_model("zai", "zai/glm-4.6", config_path=cfg_path) + assert remaining == [] + + +def test_the_same_model_in_two_spellings_is_added_once(cfg_path: Path) -> None: + from raven.config.update_providers import add_provider_model + + add_provider_model("zai", "zai/glm-4.6", config_path=cfg_path) + assert add_provider_model("zai", "glm-4.6", config_path=cfg_path) == ["zai/glm-4.6"] + + +def test_the_cli_writes_a_model_id_the_way_every_other_path_does(cfg_path: Path) -> None: + """`provider set --models` is the third write path and skipped the contract. + + It stored a bare id while the picker and the wizard stored a qualified one. + Identity still matched so nothing visibly broke -- which is how the two + spellings coexisted the last time, right up until a delete matched neither. + """ + from raven.providers.wire import stored_model_id + + set_provider_fields("anthropic", {"models": "claude-opus-4-8,anthropic/claude-sonnet-5"}, config_path=cfg_path) + + stored = _read(cfg_path)["providers"]["anthropic"]["models"] + assert stored == ["anthropic/claude-opus-4-8", "anthropic/claude-sonnet-5"] + assert stored[0] == stored_model_id("anthropic", "claude-opus-4-8") + + +# --------------------------------------------------------------------------- +# Providers that ship no default address: the probe used to call them unconfigured +# --------------------------------------------------------------------------- + + +def test_a_vendor_litellm_knows_the_address_of_is_actually_probed(cfg_path: Path) -> None: + """Ten providers carry no ``default_api_base``, and this returned + ``not_configured`` for every one of them -- telling a correctly configured + install to set the key it had already set. LiteLLM knows where four of them + live, because it is the thing that sends their requests. + """ + _seed_key(cfg_path, "groq", "sk-groq") + seen: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(str(request.url)) + return httpx.Response(401, json={"error": "bad key"}) + + result = probe_provider("groq", config_path=cfg_path, transport=_mock_transport(handler)) + + assert seen, "the probe never left the building" + assert "groq.com" in seen[0], seen + assert result["status"] == "invalid_key", "a bad key is now distinguishable from an unconfigured one" + + +def test_a_vendor_with_no_catalogue_endpoint_is_reported_as_unprobed_not_unconfigured(cfg_path: Path) -> None: + """Anthropic, OpenAI and Gemini compile the address into their SDKs, so + there is no ``/models`` to ping and nothing the user could supply. The key is + there; this probe simply cannot reach the vendor. Saying so is the honest + answer, and it is not a failure.""" + _seed_key(cfg_path, "anthropic", "sk-ant") + + def handler(request: httpx.Request) -> httpx.Response: # pragma: no cover - must not be reached + raise AssertionError(f"nothing should have been sent: {request.url}") + + result = probe_provider("anthropic", config_path=cfg_path, transport=_mock_transport(handler)) + + assert result["status"] == "no_probe_endpoint" + assert result["ok"] is False + assert "credential present" in result["error"] + + +def test_a_provider_that_genuinely_needs_an_address_still_says_so(cfg_path: Path) -> None: + """A self-hosted deployment and an Azure resource are addresses only the user + knows, so ``not_configured`` is the truth for those two -- the change must not + turn a real gap into a shrug.""" + for name in ("hosted_vllm", "azure_openai"): + _seed_key(cfg_path, name, "sk-x") + result = probe_provider(name, config_path=cfg_path, transport=_mock_transport(lambda r: httpx.Response(200))) + assert result["status"] == "not_configured", name + + +def test_a_404_from_an_address_we_guessed_is_not_reported_as_a_broken_key(cfg_path: Path) -> None: + """DeepSeek's completions endpoint is ``/beta``, which has no ``/models``. + + A 404 never says anything about a credential, so surfacing it as a failure + would be the original lie in a new spelling. A 404 from an address the *user* + supplied is different -- that is a typo they need to see -- so this only + applies where the address was derived. + """ + _seed_key(cfg_path, "deepseek", "sk-deepseek") + + derived = probe_provider( + "deepseek", + config_path=cfg_path, + transport=_mock_transport(lambda r: httpx.Response(404, json={"error": "not found"})), + ) + assert derived["status"] == "no_probe_endpoint" + + set_provider_fields("deepseek", {"api_base": "https://typo.example.com/v1"}, config_path=cfg_path) + typed = probe_provider( + "deepseek", + config_path=cfg_path, + transport=_mock_transport(lambda r: httpx.Response(404, json={"error": "not found"})), + ) + assert typed["status"] == "http_404", "a user's own wrong address must still surface" + + +def test_probing_a_login_prompting_provider_never_asks_litellm_to_resolve_it(cfg_path: Path) -> None: + """Resolving a Copilot id resolves its credentials on the way. + + With no token file that prints a device code to stdout and blocks; deriving + the address before the branch that handles Copilot separately hung this one + probe on that login. Recorded rather than raised, because the + derivation swallows exceptions to fall through -- a probe that raises is + caught and proves nothing. + """ + import litellm + + asked: list[str] = [] + + def _record(*args, **kwargs): + asked.append(str(kwargs.get("model") or (args[0] if args else "?"))) + raise Exception("unmapped") + + original = litellm.get_llm_provider + litellm.get_llm_provider = _record + try: + probe_provider("github_copilot", config_path=cfg_path, transport=_mock_transport(lambda r: httpx.Response(200))) + finally: + litellm.get_llm_provider = original + + assert not asked, f"a login-prompting id was handed to LiteLLM: {asked}" + + +@pytest.mark.parametrize("spec", [s for s in _PROVIDERS if s.is_oauth], ids=lambda s: s.name) +def test_an_oauth_provider_is_never_resolved_through_litellm_for_its_address(spec, cfg_path: Path) -> None: + """Resolving one of these resolves its credentials on the way, which prints a + device code and blocks. + + Asserted per OAuth provider rather than for the one that broke: the guard + used to be asked about the *wire* form of the id, and `wire_model` strips the + provider name outright for the codex and azure shapes -- so it was handed a + bare "probe-model" and saw nothing to object to. Today that is masked by + those providers having an address already; it would come back the moment one + did not. + """ + import litellm + + from raven.config.update_providers import _litellm_api_base + + asked: list[str] = [] + + def _record(*args, **kwargs): + asked.append(str(kwargs.get("model") or (args[0] if args else "?"))) + raise Exception("unmapped") + + original = litellm.get_llm_provider + litellm.get_llm_provider = _record + try: + assert _litellm_api_base(spec) == "" + finally: + litellm.get_llm_provider = original + + assert not asked, f"{spec.name}: handed to LiteLLM anyway ({asked})" diff --git a/tests/test_provider_auth_method.py b/tests/test_provider_auth_method.py new file mode 100644 index 00000000..891e94e2 --- /dev/null +++ b/tests/test_provider_auth_method.py @@ -0,0 +1,301 @@ +"""Whether a provider is usable, asked of every module that answers it. + +Decided in one place, ``providers.auth``. Three modules used to decide it +independently and disagreed: + +* ``config.schema._has_credentials`` gates routing -- a section it rejects is + skipped when matching a model id to a provider. +* ``config.update_providers.list_providers`` gates display -- it is what + ``raven provider list`` and the pickers show. +* ``cli._helpers.check_provider_credentials`` gates startup -- it decides + whether ``raven agent`` runs at all. + +A provider the second accepted and the first rejected was configured according +to the CLI and invisible to the router -- Gemini holding only ``api_key_list`` +read as ready in ``provider list`` and refused to start. + +(``registry.credential_kind`` is deliberately absent: it answers what shape a +provider's credentials take, not whether they are present. It is a fourth +implementation of a different question.) + +These assert the one answer, and that all three ask it. The per-implementation +records exist so that a change to any single answer is visible rather than +silently rebalancing them back into disagreement. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from raven.config.schema import Config + +#: Provider sections paired with the model id that selects them. Each case is a +#: shape of credential material, not a vendor: "the key is in the plural field", +#: "the address is set but no key", "nothing is set at all". +SCENARIOS: dict[str, dict[str, Any]] = { + "gemini_key_list_only": { + "provider": "gemini", + "model": "gemini/gemini-2.5-flash", + "section": {"apiKeyList": ["AIzaTEST"]}, + }, + "gemini_key_only": { + "provider": "gemini", + "model": "gemini/gemini-2.5-flash", + "section": {"apiKey": "AIzaTEST"}, + }, + "gemini_empty": { + "provider": "gemini", + "model": "gemini/gemini-2.5-flash", + "section": {}, + }, + "anthropic_key": { + "provider": "anthropic", + "model": "anthropic/claude-sonnet-5", + "section": {"apiKey": "sk-ant-TEST"}, + }, + "anthropic_empty": { + "provider": "anthropic", + "model": "anthropic/claude-sonnet-5", + "section": {}, + }, + "azure_key_without_base": { + "provider": "azure_openai", + "model": "azure_openai/my-deployment", + "section": {"apiKey": "az-TEST"}, + }, + "azure_key_and_base": { + "provider": "azure_openai", + "model": "azure_openai/my-deployment", + "section": {"apiKey": "az-TEST", "apiBase": "https://x.openai.azure.com"}, + }, + "ollama_base_only": { + "provider": "ollama_chat", + "model": "ollama_chat/llama3.2", + "section": {"apiBase": "http://localhost:11434"}, + }, + "ollama_empty": { + "provider": "ollama_chat", + "model": "ollama_chat/llama3.2", + "section": {}, + }, +} + + +def _config_file(tmp_path: Path, case: dict[str, Any]) -> Path: + path = tmp_path / "config.json" + path.write_text( + json.dumps( + { + "providers": {case["provider"]: case["section"]}, + "agents": {"defaults": {"model": case["model"]}}, + } + ), + encoding="utf-8", + ) + return path + + +def _routing_says(case: dict[str, Any], path: Path) -> bool: + """Would the router match this model to this provider's section?""" + config = Config.model_validate(json.loads(path.read_text(encoding="utf-8"))) + return config.get_provider(case["model"]) is not None + + +def _display_says(case: dict[str, Any], path: Path) -> bool: + """Would `raven provider list` show this provider as configured?""" + from raven.config.update_providers import list_providers + + rows = list_providers(config_path=path) + row = next((r for r in rows if r["name"] == case["provider"]), None) + return bool(row and row["configured"]) + + +def _startup_says(case: dict[str, Any], path: Path) -> bool: + """Would `raven agent` start?""" + from raven.cli._helpers import check_provider_credentials + from raven.providers.auth import MissingCredentialsError + + config = Config.model_validate(json.loads(path.read_text(encoding="utf-8"))) + try: + check_provider_credentials(config) + except MissingCredentialsError: + return False + return True + + +def _status_says(case: dict[str, Any], path: Path) -> bool: + """Would `raven status` print this provider as set up? + + Driven through the command rather than the helper it calls: this gate is a + line of formatting logic, and testing the helper would have missed it for + the same reason the agreement test missed the gate itself. + """ + from typer.testing import CliRunner + + from raven.cli.commands import app + from raven.config.loader import set_config_path + from raven.providers.registry import find_by_name + + set_config_path(path) + try: + result = CliRunner().invoke(app, ["status"]) + finally: + set_config_path(None) # type: ignore[arg-type] + + spec = find_by_name(case["provider"]) + label = (spec.label if spec else case["provider"]).lower() + for line in result.stdout.splitlines(): + if line.strip().lower().startswith(label): + return "not set" not in line + return False + + +#: Every surface that decides whether a provider is usable. A gate absent from +#: this map is a gate the agreement test cannot see -- which is how `raven +#: status` and the router's final fallback kept their own rules through a change +#: that claimed to unify them. Adding a gate means adding it here. +ANSWERS = { + "routing": _routing_says, + "display": _display_says, + "startup": _startup_says, + "status": _status_says, +} + + +@pytest.mark.parametrize("name", sorted(SCENARIOS), ids=lambda n: n) +def test_every_gate_gives_the_same_verdict(name: str, tmp_path: Path) -> None: + """One set of credentials, one verdict, whoever is asking. + + Disagreement here is always user-visible: the CLI reports a provider as + ready that the agent then refuses to start on, or the reverse. + """ + case = SCENARIOS[name] + path = _config_file(tmp_path, case) + verdicts = {who: ask(case, path) for who, ask in ANSWERS.items()} + assert len(set(verdicts.values())) == 1, f"{name}: {verdicts}" + + +def test_a_key_in_the_plural_field_is_a_configured_provider(tmp_path: Path) -> None: + """Gemini accepts a list of keys, and a list with a key in it is credentials. + + Called out separately because it is the case that shipped broken: display + said yes, routing and startup said no, so the provider appeared configured + and the agent would not run on it. + """ + case = SCENARIOS["gemini_key_list_only"] + path = _config_file(tmp_path, case) + assert _display_says(case, path) + assert _routing_says(case, path) + assert _startup_says(case, path) + + +def test_a_provider_whose_key_lives_in_a_list_sends_a_key(tmp_path: Path) -> None: + """Passing the gate is not enough; the request has to carry a credential. + + Gemini accepts several keys under one section. Reading ``api_key`` directly + at the call site sent an empty string for a section holding only the list -- + a provider that every check called configured, failing at the API instead of + at startup, which is the worst of both. + """ + case = SCENARIOS["gemini_key_list_only"] + path = _config_file(tmp_path, case) + config = Config.model_validate(json.loads(path.read_text(encoding="utf-8"))) + + provider = config.get_provider(case["model"]) + assert provider is not None + assert provider.effective_api_key == "AIzaTEST" + assert config.get_api_key(case["model"]) == "AIzaTEST" + + +def test_only_the_auth_module_decides_configuredness_from_a_key() -> None: + """No surface may read a key off a provider section to decide if it is set up. + + Six surfaces did, with six rules, and the divergence was invisible because + each looked reasonable alone. + + Matched on the syntax tree rather than on a line pattern, and on three + spellings of the read -- see ``key_reads`` for why the net is this wide. + """ + import ast + + root = Path(__file__).resolve().parents[1] / "raven" + + # Every entry is argued, because an unargued allowlist is the line-pattern + # guard again with extra steps. + allowed = { + # Not an LLM provider section: a tool's own key (deep research, media + # generation, web search), the router's, or EverOS's. + "raven/agent/loop/main.py", + "raven/agent/tools/deep_research.py", + "raven/agent/tools/media_gen.py", + "raven/agent/tools/web.py", + "raven/cli/agent_commands.py", + "raven/cli/deep_research_commands.py", + "raven/cli/gateway_commands.py", + "raven/cli/tui_commands.py", + "raven/config/update_everos.py", + "raven/config/update_tools.py", + "raven/providers/transcription.py", + # Reads a key in order to *use* it -- put it on the request, redact it + # for display, rotate it -- rather than to rule on whether a provider is + # set up. + "raven/config/schema.py", + "raven/config/update_providers.py", + "raven/providers/litellm_provider.py", + "raven/cli/_helpers.py", + "raven/cli/onboard_commands.py", + "raven/cli/provider_commands.py", + "raven/cli/status_commands.py", + "raven/tui_rpc/methods/model.py", + "raven/tui_rpc/methods/setup.py", + "raven/providers/azure_openai_provider.py", + "raven/providers/base.py", + "raven/providers/minimax_oauth_provider.py", + "raven/providers/per_model_provider.py", + # Other subsystems' credentials entirely: the skill hub, the evolver's + # judge, the EverOS memory backend, an embedding script. + "raven/config/update.py", + "raven/context_engine/factory.py", + "raven/evolver/judge/llm_client.py", + "raven/plugin/memory/everos/backend.py", + "raven/routing/generate_embeddings.py", + } + + names = {"api_key", "api_key_list", "apiKey", "apiKeyList"} + + def key_reads(tree: ast.AST) -> list[int]: + """Every read of a credential field, in any of its three spellings. + + Deliberately not narrowed to "reads in a truthiness context": every + recognizer of that context misses a shape -- an attribute read on a + passthrough section, `v.get("apiKey")` on a raw payload, the same call + inside a comprehension. + + So it flags the read and the allowlist carries the argument. A file that + legitimately touches a key says why, once, here -- which is a claim a + reviewer can check, unlike a pattern's silence. + """ + found: list[int] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Attribute) and node.attr in names: + found.append(node.lineno) + elif isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "get": + arg = node.args[0] if node.args else None + if isinstance(arg, ast.Constant) and arg.value in names: + found.append(node.lineno) + elif isinstance(node, ast.Subscript) and isinstance(node.slice, ast.Constant): + if node.slice.value in names: + found.append(node.lineno) + return found + + offenders = sorted( + f"{path.relative_to(root.parent)}:{line}" + for path in root.rglob("*.py") + if str(path.relative_to(root.parent)) not in allowed + for line in key_reads(ast.parse(path.read_text())) + ) + assert not offenders, "decide configuredness through providers.auth.credential_status: " + ", ".join(offenders) diff --git a/tests/test_provider_catalog.py b/tests/test_provider_catalog.py index 5b294bd0..71fc8582 100644 --- a/tests/test_provider_catalog.py +++ b/tests/test_provider_catalog.py @@ -256,3 +256,253 @@ def _empty_then_full() -> dict[str, tuple[str, ...]]: finally: common_models._cached_chat_models_by_provider = real # type: ignore[assignment] real.cache_clear() + + +def test_a_model_family_quirk_is_declared_not_branched_on_in_the_factory() -> None: + """The factory builds providers; it does not know which models need what. + + OpenRouter's qwen routing flag lived as an `if` there, because a fact about + one model family behind one gateway had nowhere else to go. A second such + fact would have meant a second branch. + """ + from pathlib import Path + + from raven.cli import _helpers + from raven.providers.capabilities import wire_overrides + + assert wire_overrides("openrouter", "openrouter/qwen/qwen3.7-max") == {"reasoning": {"enabled": False}} + assert wire_overrides("openrouter", "openrouter/anthropic/claude-opus-4-8") == {} + assert wire_overrides("anthropic", "anthropic/qwen-lookalike") == {}, "another provider must not inherit it" + + source = Path(_helpers.__file__).read_text(encoding="utf-8") + assert "qwen" not in source, "the model-family branch is back in the factory" + + +def test_the_bundled_label_snapshot_is_packaged() -> None: + """A data file the wheel omits is missing only for installed users. + + The build include list is a whitelist of patterns, so a new non-Python asset + is absent from the wheel by default -- and every test here runs from a source + checkout, where it is present either way. + """ + import fnmatch + import tomllib + from pathlib import Path + + from raven.providers.catalog import SNAPSHOT + + root = Path(__file__).resolve().parents[1] + assert SNAPSHOT.exists(), "the snapshot itself is missing; run scripts/refresh_models_dev_snapshot.py" + + patterns = tomllib.loads((root / "pyproject.toml").read_text(encoding="utf-8"))["tool"]["hatch"]["build"]["include"] + relative = str(SNAPSHOT.relative_to(root)) + assert any(fnmatch.fnmatch(relative, p) for p in patterns), f"{relative} matches no build include pattern" + + +#: Providers whose models the snapshot labels today. A refresh that drops one is +#: a regression no total can show: the catalogue grew from 16 providers to 37 +#: while a criterion change silently took every label off three gateways, and +#: both the provider count and the model count still went up. Taking a name off +#: this list means arguing the vendor is gone, not noticing a test went red. +LABELLED_PROVIDERS = frozenset( + { + "aihubmix", + "anthropic", + "azure_openai", + "dashscope", + "deepseek", + "gemini", + "github_copilot", + "groq", + "minimax", + "minimax_cn", + "minimax_global", + "moonshot", + "openai", + "openrouter", + "siliconflow", + "zai", + } +) + + +def _packaged_snapshot() -> dict: + import json + + from raven.providers.catalog import SNAPSHOT + + return json.loads(SNAPSHOT.read_text(encoding="utf-8")) + + +def test_the_snapshot_labels_every_provider_it_labelled_before() -> None: + snapshot = _packaged_snapshot() + missing = sorted(LABELLED_PROVIDERS - set(snapshot)) + assert not missing, f"the refresh dropped labels for: {missing}" + + empty = sorted(name for name in LABELLED_PROVIDERS if not snapshot[name].get("models")) + assert not empty, f"present but carrying no models: {empty}" + + # Present, non-empty, and every row unlabelled renders exactly like being + # absent -- the id as its own label -- so presence alone is not the property. + unlabelled = sorted( + name for name in LABELLED_PROVIDERS if not any(model.get("name") for model in snapshot[name]["models"].values()) + ) + assert not unlabelled, f"present but no model carries a name: {unlabelled}" + + +def test_the_snapshot_records_where_it_came_from() -> None: + """Provenance a reader can act on, not a date they have to trust. + + The catalogue is refreshed from someone else's repository; without the commit + it was built at, "the snapshot is stale" is unanswerable and a regenerated + file is unreviewable. + """ + source = _packaged_snapshot().get("_source") + assert source, "no _source; regenerate with scripts/refresh_models_dev_snapshot.py" + assert source.keys() >= {"repo", "ref", "sha"}, source + assert len(source["sha"]) == 40, source["sha"] + + +def test_the_snapshot_carries_labels_and_cost_and_nothing_that_shapes_a_request() -> None: + """The split this file's module docstring rests on, asserted. + + Cost prices a finished call and a stale figure costs an inaccurate total. + A context window sizes trimming and a capability flag picks a wire shape, so + both must come from the table that also routes -- carrying them here would + make a community-maintained file able to cause a wrong request. + """ + fields: set[str] = set() + for name, entry in _packaged_snapshot().items(): + if name.startswith("_"): + continue + assert set(entry) == {"models"}, f"{name} carries more than models: {sorted(entry)}" + for model in entry["models"].values(): + fields |= set(model) + assert fields <= {"name", "description", "cost"}, f"snapshot carries request-shaping fields: {sorted(fields)}" + + +def test_a_model_in_the_snapshot_is_described_and_one_outside_it_still_renders() -> None: + from raven.providers.catalog import describe + + known = describe("anthropic", "claude-sonnet-4-6") + assert known.described + assert known.label == "Claude Sonnet 4.6" + assert known.ref == "anthropic/claude-sonnet-4-6" + + # A local deployment serves whatever the user put there; no catalogue can + # know it, and the picker must still have something to show. + unknown = describe("hosted_vllm", "my-finetune-v3") + assert not unknown.described + assert unknown.label == "my-finetune-v3" + assert unknown.ref == "hosted-vllm/my-finetune-v3" + + +def test_a_stored_id_round_trips_through_describe() -> None: + """Describing an already-qualified id must not re-qualify it.""" + from raven.providers.catalog import describe + + assert describe("anthropic", "anthropic/claude-sonnet-4-6").label == "Claude Sonnet 4.6" + + +def test_what_the_user_states_about_a_model_beats_the_catalogue() -> None: + """The user naming their own deployment beats a catalogue that never heard of it. + + Only presentation. The overlay also carried `context`/`max_output` once, + justified as fixing token accounting -- nothing read them, and that + accounting already has `agents.defaults.contextWindowTokens`. + """ + from raven.config.schema import ModelOverlay + from raven.providers.catalog import SOURCE_OVERLAY, describe + + unknown = describe( + "hosted_vllm", + "my-finetune-v3", + overlay=ModelOverlay(label="Our finetune", description="tuned on support tickets"), + ) + assert unknown.described + assert unknown.source == SOURCE_OVERLAY + assert unknown.label == "Our finetune" + + # Stating one fact must not blank the others the catalogue knows. + partial = describe("anthropic", "claude-sonnet-4-6", overlay=ModelOverlay(label="Sonnet (ours)")) + assert partial.label == "Sonnet (ours)" + assert partial.description + + +def test_an_overlay_written_bare_matches_the_qualified_id() -> None: + """Overlays are matched by identity, so a pre-contract spelling still applies.""" + from raven.providers.wire import merge_key + + assert merge_key("anthropic", "claude-sonnet-4-6") == merge_key("anthropic", "anthropic/claude-sonnet-4-6") + + +# --------------------------------------------------------------------------- +# LiteLLM checks us, not the other way round +# --------------------------------------------------------------------------- + +#: Providers whose ``env_key`` deliberately differs from LiteLLM's, with the +#: argument. Adding a name here is a claim, not a way to make a test pass. +_ENV_KEY_EXEMPT: dict[str, str] = { + # A gateway speaking OpenAI's API: its key travels in OPENAI_API_KEY because + # that is the variable the driver handling the request reads. LiteLLM names + # the vendor's own variable, which nothing here sets. + "volcengine": "OPENAI_API_KEY", + # A local deployment takes an address, not a key. LiteLLM answers with the + # address variable, which is a different field of ours. + "ollama_chat": "OLLAMA_API_KEY", +} + + +def _litellm_env_keys(spec) -> list[str]: + """The variables LiteLLM would look for, or [] when it has no answer. + + Asked with the environment emptied of credentials, because the answer is + phrased as *missing* keys: on a machine that already exports the variable, + LiteLLM reports nothing missing and this test would quietly skip the provider + it was written to check. Coverage must not depend on whose laptop it runs on. + """ + import os + from unittest import mock + + from raven.providers.litellm_setup import import_litellm + + stripped = {k: v for k, v in os.environ.items() if not k.endswith(("_API_KEY", "_API_BASE", "_KEY"))} + try: + with mock.patch.dict(os.environ, stripped, clear=True): + info = import_litellm().validate_environment(model=f"{spec.model_prefix or spec.name}/probe-model") + except Exception: + return [] + return list(info.get("missing_keys") or []) + + +@pytest.mark.parametrize("spec", [s for s in PROVIDERS if s.env_key], ids=lambda s: s.name) +def test_our_env_key_is_the_one_litellm_will_read(spec) -> None: + """The registry was written by copying LiteLLM; this makes LiteLLM check it. + + A vendor renaming its variable is a silent break otherwise -- the key is set, + the request goes out without it, and the error is about authentication rather + than about a stale table. Where LiteLLM has no answer there is nothing to + compare and the case is skipped rather than assumed correct. + """ + expected = _litellm_env_keys(spec) + if not expected: + pytest.skip("LiteLLM does not name an environment variable for this provider") + + if spec.name in _ENV_KEY_EXEMPT: + assert spec.env_key == _ENV_KEY_EXEMPT[spec.name], ( + f"{spec.name}: exempted with a stated value that no longer matches the registry" + ) + return + + assert spec.env_key in expected, f"{spec.name}: we set {spec.env_key!r}, LiteLLM reads one of {expected}" + + +def test_the_env_key_exemption_list_has_no_stale_entries() -> None: + """An exemption whose divergence has gone away is a claim nobody rechecked.""" + stale = [] + for name, declared in _ENV_KEY_EXEMPT.items(): + spec = find_by_name(name) + expected = _litellm_env_keys(spec) + if expected and declared in expected: + stale.append(f"{name}: LiteLLM now names {declared!r} too -- drop the exemption") + assert not stale, "\n".join(stale) diff --git a/tests/test_provider_pin.py b/tests/test_provider_pin.py new file mode 100644 index 00000000..e8822aea --- /dev/null +++ b/tests/test_provider_pin.py @@ -0,0 +1,235 @@ +"""Tests for raven.providers.pin -- which provider a model change should pin. + +``agents.defaults.provider`` overrides what a model id says, so a stale one sends +the new model's request to the old vendor with the old vendor's key. The picker +kept the two in step; the CLI did not. The CLI then told users to edit the +field by hand -- a field no command wrote -- and the two surfaces answered the +same question differently. These assert the one rule, and that both ask it. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from raven.providers import pin + +#: (model, explicit provider, current pin) -> what must be written. +#: None means "cannot tell": the caller has to ask rather than write a guess. +#: +#: Every case runs with the named vendors configured, because the answer depends +#: on it: a pin is consulted before anything else and is answered with that +#: vendor's section whether or not it holds credentials, so pinning an +#: unconfigured one fails every request on a missing key -- never reaching the +#: fallback that lets a gateway serve a model whose id names the vendor behind it. +CASES: list[tuple[str, str, str, str | None]] = [ + # An explicit choice wins outright -- a picker selection or a --provider flag. + ("claude-sonnet-5", "anthropic", "openai", "anthropic"), + # A qualified id names its own vendor, which beats a stale pin. + ("anthropic/claude-sonnet-5", "", "openai", "anthropic"), + ("deepseek/deepseek-chat", "", "", "deepseek"), + # Prefixed but no spec of ours: keeping the pin would hand its key to a + # vendor it does not belong to. + ("mistral/mistral-large", "", "openai", "auto"), + ("mistral/mistral-large", "", "", "auto"), + # Bare, nothing pinned: no key to mis-route, so auto-detection answers. + ("some-unqualified-model", "", "", "auto"), + ("some-unqualified-model", "", "auto", "auto"), + # Bare, and the pinned provider serves it: the pin was right. + ("deepseek-chat", "", "deepseek", "deepseek"), + # Bare, and the pinned provider does not: neither keeping nor dropping it is + # safe, so the caller must ask. + ("some-model-nobody-serves", "", "deepseek", None), +] + + +#: The vendors these cases assume the user has set up. Everything the table +#: expects to be pinned must be in here, or the rule correctly answers `auto`. +CONFIGURED = {"anthropic", "deepseek", "openai"} + + +@pytest.fixture(autouse=True) +def _configured(monkeypatch): + monkeypatch.setattr(pin, "_is_configured", lambda name: name in CONFIGURED) + + +@pytest.mark.parametrize(("model", "provider", "pinned", "expected"), CASES) +def test_the_rule(model, provider, pinned, expected): + assert pin.resolve(model, provider=provider, pinned=pinned) == expected + + +def test_a_vendor_with_no_configuration_is_not_pinned(monkeypatch): + """The pin is consulted before the gateway fallback, so naming an + unconfigured vendor stops routing dead. `auto` is the answer that reaches + whichever gateway is actually serving the model.""" + monkeypatch.setattr(pin, "_is_configured", lambda name: False) + + assert pin.resolve("anthropic/claude-sonnet-5") == pin.AUTO + # An explicit choice is still the user speaking, and still wins. + assert pin.resolve("anthropic/claude-sonnet-5", provider="anthropic") == "anthropic" + + +def test_a_pin_naming_no_known_provider_answers_instead_of_raising(): + """The pinned name is free text from the config file, so it can be a typo. + + ``_is_configured`` takes a registered name and lets an unknown one raise, + but this one is whatever the file says. A misspelling that propagated as a + KeyError would abort over a config line rather than report it, so the lookup + treats an unknown section as an empty one and the answer is None: nothing + about this model can be told, which is the caller's cue to ask. + """ + assert pin.resolve("zzz-house-brand-model", provider="", pinned="opemai") is None + + +def test_a_hand_typed_id_at_a_custom_endpoint_is_not_routed_by_its_name(): + """The pin is what makes an unqualified id safe, so it is not optional. + + The wizard qualifies ids it read from a provider's own ``/v1/models``, but a + custom endpoint has the user type one, and what they type is the name their + server answers to -- often a name some other vendor also uses. Routed by + keyword, "gpt-4o" against a local server reaches OpenAI and spends OpenAI's + key. The pin decides first, so the section just configured is the one that + answers, whatever the id happens to be called. + """ + for typed in ("gpt-4o", "claude-sonnet-5", "my-local-model"): + assert pin.resolve(typed, provider="custom", pinned="") == "custom", typed + + +def test_a_local_deployment_keeps_its_pin_without_being_asked(): + """Its server names whatever models it likes and there is no key to mis-route, + so a bare id under a local pin is that deployment's.""" + from raven.providers.registry import PROVIDERS + + local = next(spec.name for spec in PROVIDERS if spec.is_local) + assert pin.resolve("whatever-it-serves", pinned=local) == local + + +# --------------------------------------------------------------------------- +# The property that made this a module: both surfaces write the same pair. +# --------------------------------------------------------------------------- + + +def _cli_writes(tmp_path: Path, model: str, provider: str, pinned: str) -> tuple[str, str] | None: + from typer.testing import CliRunner + + from raven.cli.commands import app + + config = tmp_path / "config.json" + config.write_text(json.dumps({"agents": {"defaults": {"provider": pinned}}}), encoding="utf-8") + + args = ["provider", "use", model] + (["--provider", provider] if provider else []) + result = CliRunner().invoke(app, args) + if result.exit_code != 0: + return None + defaults = json.loads(config.read_text(encoding="utf-8"))["agents"]["defaults"] + return defaults["model"], defaults.get("provider", "") + + +def _tui_writes(tmp_path: Path, model: str, provider: str, pinned: str) -> tuple[str, str] | None: + from raven.tui_rpc.errors import ConfigValidationError + from raven.tui_rpc.methods import config as config_methods + + config = tmp_path / "config.json" + config.write_text(json.dumps({"agents": {"defaults": {"provider": pinned}}}), encoding="utf-8") + + params = {"key": "model", "value": model} + if provider: + params["provider"] = provider + try: + config_methods._set_model(params, model, None) + except ConfigValidationError: + return None + defaults = json.loads(config.read_text(encoding="utf-8"))["agents"]["defaults"] + return defaults["model"], defaults.get("provider", "") + + +@pytest.mark.parametrize(("model", "provider", "pinned", "expected"), CASES) +def test_the_cli_and_the_picker_write_the_same_pair(monkeypatch, tmp_path, model, provider, pinned, expected): + """Same inputs, same ``(model, provider)`` on disk -- or both refuse. + + Asserted as agreement between the two surfaces rather than each against a + table, because the failure being prevented is precisely that one of them + quietly grows a rule the other does not have. + """ + from raven.config.loader import set_config_path + + path = tmp_path / "config.json" + set_config_path(path) + monkeypatch.setattr("raven.tui_rpc.methods.config._config_path", lambda: path) + monkeypatch.setattr("raven.config.update.get_config_path", lambda: path) + + cli = _cli_writes(tmp_path, model, provider, pinned) + tui = _tui_writes(tmp_path, model, provider, pinned) + + assert cli == tui, f"{model!r} (provider={provider!r}, pinned={pinned!r}): CLI wrote {cli}, picker wrote {tui}" + if expected is None: + assert cli is None, "an id nobody can place must be refused, not written" + else: + assert cli is not None and cli[1] == expected + + +# --------------------------------------------------------------------------- +# Every writer, not just the one that was fixed +# --------------------------------------------------------------------------- + + +def test_no_surface_writes_the_default_model_without_deciding_its_pin(): + """The hole was fixed at one call site and stayed open at four others. + + ``raven provider use`` was made to write ``agents.defaults.provider``, and + the warning that a stale pin makes a switch ineffective was deleted on the + grounds that it had become impossible. It had not: onboarding wrote the model + through its own helper and left the pin alone, so finishing the wizard on + Anthropic while DeepSeek was pinned routed Anthropic's model to DeepSeek -- + with DeepSeek's key. A rule enforced at one caller is not enforced. + + Scanned rather than asserted per call site, because the next writer is the + one nobody thought of -- and **both spellings count**. An earlier version of + this guard looked only for ``set_default_model`` and was therefore blind to + ``tui_rpc/methods/config.py``, which writes the same field through + ``_set_nested`` and happens to be correct. + """ + import ast + import pathlib + + root = pathlib.Path(__file__).resolve().parents[1] / "raven" + definition = root / "config" / "update.py" + offenders: list[str] = [] + + for path in sorted(root.rglob("*.py")): + if path == definition: + continue # the function itself + source = path.read_text(encoding="utf-8") + tree = ast.parse(source) + # A *write* of the pin, not a mention of it. Keying on the string alone + # was satisfied by the `_get_nested(payload, "agents.defaults.provider")` + # read two lines above the write, so this exempted the very file its + # docstring names -- deleting both pin writes there left it green. + writes_pin = any( + isinstance(call, ast.Call) + and any(isinstance(a, ast.Constant) and a.value == "agents.defaults.provider" for a in call.args) + and (call.func.attr if isinstance(call.func, ast.Attribute) else getattr(call.func, "id", "")) + in {"_set_nested", "set_nested"} + for call in ast.walk(tree) + ) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + name = node.func.attr if isinstance(node.func, ast.Attribute) else getattr(node.func, "id", "") + + if name == "set_default_model": + if not any(kw.arg == "provider" for kw in node.keywords): + offenders.append(f"{path.relative_to(root.parent)}:{node.lineno} (set_default_model)") + continue + + # The other spelling: a raw nested write of the same key. + targets = [a.value for a in node.args if isinstance(a, ast.Constant) and isinstance(a.value, str)] + if "agents.defaults.model" in targets and not writes_pin: + offenders.append(f"{path.relative_to(root.parent)}:{node.lineno} (raw key write)") + + assert not offenders, ( + "these write the model and leave the pin to whatever it was; decide it with " + "providers.pin.resolve and write both:\n" + "\n".join(offenders) + ) diff --git a/tests/test_provider_prompt_cache.py b/tests/test_provider_prompt_cache.py new file mode 100644 index 00000000..fccab46e --- /dev/null +++ b/tests/test_provider_prompt_cache.py @@ -0,0 +1,546 @@ +"""Tests for raven.providers.prompt_cache -- who may carry cache_control. + +The decision used to exist three times: once in the provider that builds the +request and once in each token strategy that places breakpoints. They disagreed +in the way copies do -- the provider's only ever marked the system message and +the tool list, so it could not have answered for the marks the strategies stamp +onto the last conversation message, which is where a doubled Gemini bill came +from. These assert the one answer, and that all three ask it. +""" + +from __future__ import annotations + +import pytest + +from raven.providers import prompt_cache + + +@pytest.fixture(autouse=True) +def _forget_suppressions(): + prompt_cache.reset_suppressions() + yield + prompt_cache.reset_suppressions() + + +# --- The predicate: (wire x model family) --- + + +@pytest.mark.parametrize( + ("model", "expected", "why"), + [ + ("anthropic/claude-fable-5", True, "direct to the vendor whose API defines the field"), + ("openrouter/anthropic/claude-fable-5", True, "a wire that carries it, a vendor that reads it"), + ("openrouter/google/gemini-3.5-flash", False, "carried, forwarded, and billed twice"), + ("openrouter/qwen/qwen3.7-max", False, "carried, and it cost the model its own auto-caching"), + ("openrouter/deepseek/deepseek-chat", False, "DeepSeek caches automatically and takes no breakpoints"), + ("deepseek/deepseek-chat", False, "an OpenAI-shaped wire has nowhere to put it"), + ("siliconflow/anthropic/claude-fable-5", False, "the gateway's wire decides, and this one cannot carry it"), + ("", False, "no id, no answer"), + ], +) +def test_the_answer_is_the_wire_and_the_family_together(model, expected, why): + assert prompt_cache.accepts_cache_control(model) is expected, why + + +def test_the_measured_regression_is_the_one_that_changed(): + """The three answers this rule was measured against on a real machine. + + Fable through the gateway cached only with the field, so it must stay True. + A marked prompt billed Gemini for nearly double its tokens, and cost Qwen + its own automatic caching, so both must become False. A change that flips + any of these three is a change to a real bill. + """ + assert prompt_cache.accepts_cache_control("openrouter/anthropic/claude-fable-5") is True + assert prompt_cache.accepts_cache_control("openrouter/google/gemini-3.5-flash") is False + assert prompt_cache.accepts_cache_control("openrouter/qwen/qwen3.7-max") is False + + +def test_every_place_that_marks_a_request_asks_the_same_question(): + """The property the three copies did not have. + + Asserted as agreement rather than as "they all import it", because importing + one answer and then adjusting it locally is exactly how the copies drifted. + """ + from raven.providers.litellm_provider import LiteLLMProvider + from raven.token_wise import cache_optimizer, system_and_tail_cache + + # Stored ids, which is what production carries: each names the provider + # serving it, so the strategies -- which only ever see the id -- have the + # same information the provider has. + stored = { + "openrouter": ("openrouter/anthropic/claude-fable-5", "openrouter/google/gemini-3.5-flash"), + "anthropic": ("anthropic/claude-fable-5",), + "deepseek": ("deepseek/deepseek-chat",), + "siliconflow": ("siliconflow/anthropic/claude-fable-5",), + } + for provider_name, models in stored.items(): + provider = LiteLLMProvider(api_key="", default_model="x", provider_name=provider_name) + for model in models: + expected = prompt_cache.accepts_cache_control(model) + assert provider._supports_cache_control(model) is expected, model + assert cache_optimizer._supports_cache_control(model) is expected, model + assert system_and_tail_cache._supports_cache_control(model) is expected, model + + +# --- Learned suppression --- + + +def test_a_model_an_upstream_refused_is_not_marked_again(): + model = "openrouter/anthropic/claude-3-haiku" + assert prompt_cache.accepts_cache_control(model) is True + + prompt_cache.suppress(model) + + assert prompt_cache.is_suppressed(model) + assert prompt_cache.accepts_cache_control(model) is False + + +def test_suppressing_one_model_does_not_touch_its_neighbours(): + prompt_cache.suppress("openrouter/anthropic/claude-3-haiku") + assert prompt_cache.accepts_cache_control("openrouter/anthropic/claude-fable-5") is True + + +#: Two refusal shapes captured verbatim from `openrouter/anthropic/claude-3-haiku`. +#: Both arrive for the same model, because the gateway picks a different +#: upstream per request -- and the second, which never names the field, is the +#: one a matcher built on the field name misses, reading as an intermittent +#: failure. +_REFUSAL_NAMING_THE_FIELD = ( + "Error calling LLM: litellm.BadRequestError: OpenrouterException - " + "messages.0.content.0.text.cache_control: Extra inputs are not permitted" +) +_REFUSAL_FROM_THE_UPSTREAM = ( + "Error calling LLM: litellm.BadRequestError: OpenrouterException - " + '{"error":{"message":"Provider returned error","code":400,"metadata":{"raw":' + '"{\\"message\\":\\"You invoked an unsupported model or your request did not allow ' + 'prompt caching. See the documentation for more information.\\"}",' + '"provider_name":"Amazon Bedrock","is_byok":false}}}' +) + + +@pytest.mark.parametrize( + ("message", "expected"), + [ + (_REFUSAL_NAMING_THE_FIELD, True), + (_REFUSAL_FROM_THE_UPSTREAM, True), + ("Error calling LLM: invalid_request_error: cache_control is not supported", True), + ("BadRequestError: 400 context_length_exceeded", False), + # The status has to be its own token. As a bare substring it also matched + # the "400" inside "1400ms", so a rate limit or a timeout whose text + # happened to name the field read as a refusal -- and that costs the + # model its caching for the rest of the process, quietly. + # A gateway paraphrasing its upstream can drop the numeric code entirely, + # and the spelling that arrives carries a space -- which the run-together + # forms do not match, leaving the status as the only detector. + ("Bad Request: your request did not allow prompt caching", True), + ("BAD REQUEST -- cache_control: Extra inputs are not permitted", True), + ("429 rate limited, retry after 1400ms; cache_control was fine", False), + ("Timeout after 24000ms while streaming a prompt caching request", False), + ("Timeout while sending a cache_control payload", False), + ("500 Provider returned error: prompt caching is temporarily unavailable", False), + ("429 rate limited", False), + ("", False), + ], +) +def test_only_a_refusal_naming_the_field_counts(message, expected): + """Both halves required. + + The name alone would read a timeout whose payload was logged as a dialect + problem and switch caching off for the process; the status alone would + swallow every other malformed request into a silent retry. + """ + assert prompt_cache.is_rejection(message) is expected + + +# --- Stripping marks a strategy already placed --- + + +def test_stripping_removes_every_breakpoint_a_strategy_placed(): + """Needed as well as suppression, not instead of it: the strategies mark the + payload upstream of the provider, so the marks are already in the messages a + retry would resend.""" + messages = [ + {"role": "system", "content": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": "hi", "cache_control": {"type": "ephemeral"}}, + {"role": "assistant", "content": [{"type": "text", "text": "ok"}]}, + ] + tools = [{"name": "a"}, {"name": "b", "cache_control": {"type": "ephemeral"}}] + + stripped_messages, stripped_tools = prompt_cache.strip(messages, tools) + + assert "cache_control" not in str(stripped_messages) + str(stripped_tools) + # Content survives, and a one-element text block collapses back to the string + # it was wrapped from -- the wrap exists only to hold the breakpoint. + assert stripped_messages[0]["content"] == "sys" + assert stripped_messages[1]["content"] == "hi" + assert stripped_messages[2]["content"] == "ok" + assert [t["name"] for t in stripped_tools] == ["a", "b"] + + +def test_stripping_leaves_the_caller_s_list_alone(): + """The retry re-sends what it was given; mutating it would change what a + caller holding the same list believes it sent.""" + messages = [{"role": "user", "content": "hi", "cache_control": {"type": "ephemeral"}}] + + prompt_cache.strip(messages, None) + + assert messages[0]["cache_control"] == {"type": "ephemeral"} + + +def test_stripping_tolerates_no_tools_and_odd_blocks(): + messages = [{"role": "user", "content": None}, {"role": "tool", "content": ["not-a-dict"]}] + + stripped, tools = prompt_cache.strip(messages, None) + + assert tools is None + assert stripped[0]["content"] is None + assert stripped[1]["content"] == ["not-a-dict"] + + +# --- The retry that learns it (raven.providers.base) --- + + +class _RecordingProvider: + """A provider whose first call refuses the field and whose second succeeds. + + Built from ``LLMProvider`` rather than mocked at the transport, because the + behaviour under test spans two layers: the retry strips what a strategy + already placed, and the provider must not put its own back on the way out. + """ + + def __init__(self, errors: list[str]): + from raven.providers.base import LLMProvider, LLMResponse + + self._errors = errors + self._LLMResponse = LLMResponse + self.sent: list[tuple[list, list | None]] = [] + + outer = self + + class _P(LLMProvider): + async def chat(self, messages, tools=None, model=None, **kwargs): + outer.sent.append((messages, tools)) + if outer._errors: + return LLMResponse(content=outer._errors.pop(0), finish_reason="error") + return LLMResponse(content="ok", finish_reason="stop") + + def get_default_model(self) -> str: + return "openrouter/anthropic/claude-3-haiku" + + self.provider = _P(api_key="test") + + +def _marked_payload(): + return ( + [{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}], + [{"name": "t", "cache_control": {"type": "ephemeral"}}], + ) + + +@pytest.mark.asyncio +async def test_a_refused_field_is_dropped_and_the_turn_retried_immediately(): + """The failure this exists for: OpenRouter routes an Anthropic model to + Bedrock, which wants ``cachePoint``, and answers 400 every single turn. + + The retry has to strip as well as suppress -- a token strategy placed these + marks before the provider was ever called, so suppression alone would resend + exactly what was refused. + """ + model = "openrouter/anthropic/claude-3-haiku" + rig = _RecordingProvider(["litellm.BadRequestError: 400 tools.16.cache_control: Extra inputs are not permitted"]) + messages, tools = _marked_payload() + + response = await rig.provider.chat_with_retry(messages=messages, tools=tools, model=model) + + assert response.finish_reason == "stop" + assert len(rig.sent) == 2, "expected exactly one extra attempt" + assert "cache_control" in str(rig.sent[0]), "the first attempt should carry what the strategy placed" + assert "cache_control" not in str(rig.sent[1]), "the retry resent the field that was just refused" + assert prompt_cache.is_suppressed(model) + + +@pytest.mark.asyncio +async def test_an_unrelated_bad_request_is_not_retried_or_learned_from(): + """Nothing is swallowed: an error that does not name the field surfaces as + itself, and no model is marked as refusing anything.""" + model = "openrouter/anthropic/claude-3-haiku" + rig = _RecordingProvider(["litellm.BadRequestError: 400 context_length_exceeded"] * 8) + messages, tools = _marked_payload() + + response = await rig.provider.chat_with_retry(messages=messages, tools=tools, model=model) + + assert response.finish_reason == "error" + assert "context_length_exceeded" in response.content + assert not prompt_cache.is_suppressed(model) + + +@pytest.mark.asyncio +async def test_the_field_is_dropped_once_not_on_every_attempt(): + """A model that refuses the field and then keeps failing must not spend its + whole retry ladder re-learning the same thing.""" + model = "openrouter/anthropic/claude-3-haiku" + refusal = "BadRequestError: 400 cache_control not permitted" + rig = _RecordingProvider([refusal] * 8) + messages, tools = _marked_payload() + + response = await rig.provider.chat_with_retry(messages=messages, tools=tools, model=model) + + assert response.finish_reason == "error" + assert prompt_cache.is_suppressed(model) + # Attempt 1 carried the marks, the rest did not -- and the ladder was not + # restarted, so the total stays inside the normal budget. + assert "cache_control" in str(rig.sent[0]) + assert all("cache_control" not in str(sent) for sent in rig.sent[1:]) + + +# --- Nobody marks a request without asking --- + + +def _production_files(): + import pathlib + + root = pathlib.Path(__file__).resolve().parents[1] / "raven" + return sorted(p for p in root.rglob("*.py") if "__pycache__" not in p.parts) + + +def test_the_fields_value_has_one_definition(): + """Three modules each spelled out ``{"type": "ephemeral"}``. + + Anthropic's is the only shape today, so three copies agreed by luck rather + than by construction -- and a fourth would be written by whoever adds the + next marker. + """ + offenders = [ + str(path) + for path in _production_files() + if "ephemeral" in path.read_text(encoding="utf-8") + and path.name != "prompt_cache.py" + and '"type": "ephemeral"' in path.read_text(encoding="utf-8") + ] + assert not offenders, "import CACHE_CONTROL from providers.prompt_cache:\n" + "\n".join(offenders) + + +def test_every_module_that_writes_the_field_asks_whether_it_may(): + """The shape that would slip past every other test here. + + A fourth marker -- a new strategy, a new provider backend -- can place + breakpoints correctly, agree with nothing, and be found only by a bill. The + three that exist are listed because each was read and each asks; a new name + on this list is a claim that it does too. + """ + import ast + + writers = { + "raven/providers/prompt_cache.py", # the answer itself + "raven/providers/litellm_provider.py", + "raven/token_wise/cache_optimizer.py", + "raven/token_wise/system_and_tail_cache.py", + } + root = _production_files()[0].parents[1] + found = set() + for path in _production_files(): + source = path.read_text(encoding="utf-8") + if '"cache_control"' not in source and "'cache_control'" not in source: + continue + rel = str(path.relative_to(root)) + found.add(rel) + if rel == "raven/providers/prompt_cache.py": + continue + # The import, not the spelling: a local helper that happens to be called + # `_supports_cache_control` and answers on its own would satisfy a text + # scan while being exactly the second copy this is here to prevent. + imported = any( + isinstance(node, ast.ImportFrom) + and node.module == "raven.providers.prompt_cache" + and any(alias.name == "accepts_cache_control" for alias in node.names) + for node in ast.walk(ast.parse(source)) + ) + assert imported, f"{rel} marks requests without importing the answer" + + assert found <= writers, f"unreviewed writers of the field: {sorted(found - writers)}" + + +@pytest.mark.asyncio +async def test_the_client_takes_off_marks_meant_for_a_vendor_it_is_not_calling(): + """A strategy sees an id; only the client knows where the request goes. + + ``anthropic/claude-3`` served through an OpenAI-shaped gateway is a shape the + config matcher produces on purpose -- an id naming a vendor, routed to + whoever actually has credentials. The strategy marks it (the id says + Anthropic), the wire has nowhere to carry the field, and the vendor behind + the gateway either refuses it or bills the prompt twice without saying so. + The last word belongs to whoever sends it. + """ + from raven.providers.litellm_provider import LiteLLMProvider + from raven.token_wise.cache_optimizer import CacheOptimizer + + model = "anthropic/claude-3" + messages, tools, _ = await CacheOptimizer().before_llm_call( + [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}], None, model + ) + assert "cache_control" in str(messages), "premise: the strategy marks this id" + + client = LiteLLMProvider(api_key="k", default_model=model, provider_name="aihubmix") + assert client._supports_cache_control(model) is False + + sent, sent_tools = prompt_cache.strip(messages, tools) + assert "cache_control" not in str(sent) + str(sent_tools) + + +def test_a_client_that_may_carry_the_field_still_gets_the_strategys_marks(): + """Stripping is for the disagreement, not a blanket removal: where the client + and the strategy agree, the breakpoints the strategy placed must survive.""" + from raven.providers.litellm_provider import LiteLLMProvider + + client = LiteLLMProvider(api_key="k", default_model="x", provider_name="openrouter") + assert client._supports_cache_control("openrouter/anthropic/claude-fable-5") is True + + +@pytest.mark.asyncio +async def test_marking_then_stripping_returns_the_payload_it_started_from(): + """Removing the field is not undoing the marking. + + To have somewhere to put a breakpoint the strategy rewrites string content + into a one-element text block. Taking the key back off left that rewrite in + place, so a wire judged unable to carry the field was still sent an + Anthropic-shaped payload -- and "content must be a string" is among the + commonest ways an OpenAI-compatible endpoint refuses. That refusal names + neither the field nor prompt caching, so `is_rejection` would not learn from + it either: the risk moved from "field rejected" to "shape rejected", with no + fallback behind it. + """ + from raven.token_wise.cache_optimizer import CacheOptimizer + + messages = [ + {"role": "system", "content": "S"}, + {"role": "user", "content": "U"}, + {"role": "assistant", "content": "", "tool_calls": [{"id": "1"}]}, + {"role": "tool", "content": "TOOL RESULT"}, + ] + tools = [{"name": "t"}] + + marked, marked_tools, _ = await CacheOptimizer().before_llm_call( + [dict(m) for m in messages], list(tools), "openrouter/anthropic/claude-fable-5" + ) + assert "cache_control" in str(marked), "premise: the strategy marks this model" + + assert prompt_cache.strip(marked, marked_tools) == (messages, tools) + + +def test_stripping_leaves_content_that_was_already_a_list_alone(): + """The collapse undoes one specific rewrite, not every list. + + Multi-block content and blocks carrying anything besides type/text are the + caller's own shape and must survive untouched. + """ + messages = [ + {"role": "user", "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]}, + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": "x"}}]}, + ] + + stripped, _ = prompt_cache.strip(messages, None) + + assert stripped == messages + + +def test_a_refusal_whose_str_is_only_a_status_is_still_recognised(): + """The client paraphrases the gateway, which paraphrased the upstream. + + LiteLLM's streaming path raises `MaskedHTTPStatusError`, whose `str()` is a + URL and a status code and names nothing -- while the body it was built from + sits on `.text`. Matching only the rendered string made the same refusal + learnable on the non-streaming path and invisible on the streaming one, which + is the path the TUI uses. + """ + + class _Masked(Exception): + def __init__(self): + super().__init__("Client error '400 Bad Request' for url 'https://openrouter.ai/api/v1/chat/completions'") + self.text = ( + '{"error":{"message":"messages.0.content.0.text.cache_control: Extra inputs are not permitted"}}' + ) + + masked = _Masked() + assert "cache_control" not in str(masked), "premise: the rendered string says nothing" + assert prompt_cache.is_rejection(masked) is True + + +def test_an_exception_that_says_nothing_anywhere_is_not_a_refusal(): + class _Opaque(Exception): + text = "Client error '400 Bad Request' for url 'https://example/x'" + + assert prompt_cache.is_rejection(_Opaque("boom")) is False + + +@pytest.mark.asyncio +async def test_the_non_streaming_retry_also_reads_the_body_off_the_exception(): + """Same asymmetry, other path. + + Whether ``str(exc)`` carries the response body is a property of the handler + that raised it, not of streaming versus not. The fix landed on the streaming + path first and the non-streaming one went on matching a rendered string with + the exception sitting in the same scope. + """ + from raven.providers import prompt_cache + from raven.providers.base import LLMProvider + + prompt_cache.reset_suppressions() + + class _Masked(Exception): + def __init__(self): + super().__init__("Client error '400 Bad Request' for url 'https://example/x'") + self.text = ( + '{"error":{"message":"messages.0.content.0.text.cache_control: Extra inputs are not permitted"}}' + ) + + sent: list[bool] = [] + + class _P(LLMProvider): + async def chat(self, messages, tools=None, model=None, **kwargs): + sent.append("cache_control" in str(messages)) + if len(sent) == 1: + raise _Masked() + from raven.providers.base import LLMResponse + + return LLMResponse(content="ok", finish_reason="stop") + + def get_default_model(self) -> str: + return "openrouter/anthropic/claude-3-haiku" + + provider = _P(api_key="k") + messages = [{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}] + + response = await provider.chat_with_retry(messages=messages, model="openrouter/anthropic/claude-3-haiku") + + assert response.finish_reason == "stop" + assert sent == [True, False], f"expected one marked attempt then one clean retry, got {sent}" + assert prompt_cache.is_suppressed("openrouter/anthropic/claude-3-haiku") + prompt_cache.reset_suppressions() + + +def test_the_refusal_verdict_survives_a_provider_that_swallows_the_exception(): + """`LiteLLMProvider.chat` turns the exception into a string before the retry + layer sees it, so asking there asks about a paraphrase. + + Deciding it in `classify_error` -- next to every other verdict, while the + exception is alive -- is what makes the non-streaming path able to learn from + a refusal whose `str()` says nothing. Both spellings must reach the same + answer: the live exception, and the string a provider left behind. + """ + from raven.providers.base import LLMProvider + + class _Masked(Exception): + def __init__(self): + super().__init__("Client error '400 Bad Request' for url 'https://x'") + self.text = ( + '{"error":{"message":"messages.0.content.0.text.cache_control: Extra inputs are not permitted"}}' + ) + + assert LLMProvider.classify_error(_Masked()).refuses_prompt_cache is True + swallowed = "Error calling LLM: litellm.BadRequestError: 400 cache_control: Extra inputs are not permitted" + assert LLMProvider.classify_error(None, swallowed).refuses_prompt_cache is True + + # And nothing else becomes a refusal on the way. + assert LLMProvider.classify_error(None, "Error calling LLM: 400 context_length_exceeded").refuses_prompt_cache is ( + False + ) diff --git a/tests/test_provider_rates.py b/tests/test_provider_rates.py new file mode 100644 index 00000000..099c8410 --- /dev/null +++ b/tests/test_provider_rates.py @@ -0,0 +1,629 @@ +"""Tests for raven.providers.rates -- what a model costs and how much it holds. + +Split out of ``test_token_wise_pricing.py`` when the resolution ladder moved into +``raven.providers``: these exercise where a number comes from, while the cost +arithmetic on top of it stays with the module that does the arithmetic. +""" + +from __future__ import annotations + +import json +import time + +import httpx +import pytest + +from raven.providers import model_catalog_cache, rates +from raven.providers.litellm_setup import import_litellm +from raven.providers.rates import ( + _FALLBACK_PRICING, + resolve_context_window, + token_rates, +) + +# Imported here, at collection, and not left to whichever test asks for it first: +# `_patch_openrouter` replaces `httpx.Client` process-wide, and LiteLLM builds +# clients on the way up. A first import under that patch leaves the module +# half-initialised, and every later `import litellm` -- including the ones inside +# these tests -- gets back the broken one with a circular-import AttributeError. +import_litellm() + +# The real fetch, captured before conftest's autouse guard stubs it to {}. +_REAL_FETCH = rates._fetch_openrouter_models + + +@pytest.fixture(autouse=True) +def _reset_catalog_state(): + rates._OPENROUTER_CACHE.clear() + yield + rates._OPENROUTER_CACHE.clear() + + +def _patch_openrouter(monkeypatch, handler): + """Route the real OpenRouter fetch through a MockTransport. + + Restores the real ``_fetch_openrouter_models`` (conftest stubs it to {} so no + test hits the network by default), then mocks the httpx transport. Returns a + counter dict whose ``["calls"]`` tracks network hits. + """ + counter = {"calls": 0} + + def counting_handler(request): + counter["calls"] += 1 + return handler(request) + + transport = httpx.MockTransport(counting_handler) + real_client = httpx.Client + + def client_factory(*args, **kwargs): + kwargs.setdefault("transport", transport) + return real_client(*args, **kwargs) + + monkeypatch.setattr(rates, "_fetch_openrouter_models", _REAL_FETCH) + monkeypatch.setattr(rates.httpx, "Client", client_factory) + monkeypatch.setattr(rates, "_OPENROUTER_CACHE_TIME", 0.0) + return counter + + +def _models_response(models): + return httpx.Response(200, content=json.dumps({"data": models})) + + +def _rate_cost(model, input_tokens, output_tokens): + """Rates applied to token counts, so a tier can be asserted as a figure.""" + pair = token_rates(model, input_tokens, output_tokens) + return None if pair is None else input_tokens * pair[0] + output_tokens * pair[1] + + +_DEEPSEEK_MODELS = [ + { + "id": "deepseek/deepseek-v4-pro", + "context_length": 163840, + "pricing": {"prompt": "0.0000005", "completion": "0.0000015"}, + } +] + + +# --- The tier LiteLLM answers (see `rates.token_rates` for the order) --- + + +def test_a_litellm_mapped_model_is_priced_without_touching_the_network(monkeypatch): + counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + assert token_rates("anthropic/claude-sonnet-4-5", 1000, 500) is not None + assert counter["calls"] == 0 + + +def test_a_model_no_tier_knows_has_no_rates(): + assert token_rates("nonexistent-vendor/imaginary-model-9000", 100, 100) is None + + +# --- The tier the bundled snapshot answers, keyed by provider --- + + +def test_a_vendor_litellm_does_not_price_is_answered_by_its_own_published_rate(monkeypatch): + """The tier that exists so a direct route is not priced off a gateway's table. + + ``zai/glm-5.2`` was reported at OpenRouter's figure because the live catalogue + happened to carry a model of that name. The snapshot is keyed by provider, so + Z.ai's own published price is the one that answers. + """ + from raven.providers.catalog import model_cost + + published = model_cost("zai/glm-5.2") + if not published: + pytest.skip("the snapshot no longer carries this model") + + counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + pair = rates._try_snapshot_rates("zai/glm-5.2") + + assert pair == (published["input"] / 1e6, published["output"] / 1e6) + assert counter["calls"] == 0, "the vendor's own row needs no live catalogue" + + +def test_the_snapshot_tier_is_silent_about_a_model_it_does_not_carry(): + assert rates._try_snapshot_rates("nonexistent-vendor/imaginary-model-9000") is None + + +# --- The tier OpenRouter's live catalogue answers, for ids naming it --- + + +def test_an_openrouter_model_litellm_lags_on_is_priced_live(monkeypatch): + _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + cost = _rate_cost("openrouter/deepseek/deepseek-v4-pro", 1000, 500) + + assert cost == pytest.approx(1000 * 0.0000005 + 500 * 0.0000015, rel=1e-9) + + +def test_a_bare_id_under_openrouter_resolves_to_the_same_row(monkeypatch): + """Within OpenRouter's own namespace a bare id names the model the full one + does, so the alias stays. What was removed is asking this table at all about a + request that does not go to OpenRouter.""" + _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + assert _rate_cost("openrouter/deepseek-v4-pro", 1000, 0) == pytest.approx(1000 * 0.0000005, rel=1e-9) + + +def test_an_openrouter_model_absent_from_the_table_has_no_rates(monkeypatch): + _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + assert token_rates("openrouter/some/model-not-listed", 1000, 500) is None + + +def test_a_network_failure_falls_through_to_the_bundled_copy(monkeypatch): + """Offline, the ladder keeps going rather than inventing a number. + + The live tier degrades to nothing and the bundled snapshot answers, which is + the whole reason a copy ships: a figure from the last refresh beats no figure + at all for a total nobody can act on. Nothing is fabricated -- a model absent + from every tier is still None. + """ + from raven.providers.catalog import model_cost + + def boom(req): + raise httpx.ConnectError("offline") + + _patch_openrouter(monkeypatch, boom) + + published = model_cost("openrouter/deepseek/deepseek-v4-pro") + if published: + assert token_rates("openrouter/deepseek/deepseek-v4-pro", 1000, 500) == ( + published["input"] / 1e6, + published["output"] / 1e6, + ) + assert token_rates("openrouter/nobody/has-heard-of-this", 1000, 500) is None + + +def test_the_live_table_is_fetched_once_and_reused(monkeypatch): + counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + token_rates("openrouter/deepseek/deepseek-v4-pro", 1000, 500) + token_rates("openrouter/deepseek/deepseek-v4-pro", 10, 20) + + assert counter["calls"] == 1 + + +_UNMAPPED_CATALOG = [ + { + "id": "fakevendor/imaginary-priced-9000", + "context_length": 163840, + "pricing": {"prompt": "0.0000005", "completion": "0.0000015"}, + } +] + + +def test_a_model_that_does_not_route_through_openrouter_never_reads_its_table(monkeypatch): + """The bare-name hijack, asserted from the other side. + + OpenRouter's table used to answer for every id LiteLLM missed. It is a + cross-vendor catalogue, so a self-hosted deployment or a direct vendor route + was reported at somebody else's price -- and, worse, somebody else's context + window. An id that does not name OpenRouter must not reach it at all. + """ + counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_UNMAPPED_CATALOG)) + + assert token_rates("fakevendor/imaginary-priced-9000", 1000, 500) is None + assert counter["calls"] == 0, "the live catalogue was consulted for a non-OpenRouter id" + + +def test_the_hijacked_model_reports_neither_a_price_nor_a_window(monkeypatch): + """The case this was filed for: a self-hosted vLLM borrowed a hosted model's + numbers because the bare name matched. Unknown is the correct answer -- the + caller keeps its configured window rather than trimming to a stranger's.""" + _patch_openrouter(monkeypatch, lambda req: _models_response(_UNMAPPED_CATALOG)) + + assert token_rates("hosted_vllm/qwen3-32b") is None + assert resolve_context_window("hosted_vllm/qwen3-32b") is None + + +# --- The manual table, last --- + + +def test_the_manual_table_answers_a_model_too_new_for_the_others(): + model = next(iter(_FALLBACK_PRICING)) + p_rate, c_rate = _FALLBACK_PRICING[model] + + assert _rate_cost(model, 1000, 500) == pytest.approx(1000 * p_rate + 500 * c_rate, rel=0.01) + + +# --- Context windows --- + + +def _patch_litellm_info(monkeypatch, fn): + """Stub litellm.get_model_info (offline) -- fn(model) returns a dict or raises.""" + import litellm + + monkeypatch.setattr(litellm, "get_model_info", fn) + + +def _litellm_miss(_model): + raise Exception("This model isn't mapped yet") + + +def _patch_litellm_blind(monkeypatch): + """Make LiteLLM miss for real: the price table is consulted before the ask. + + Stubbing ``get_model_info`` alone stopped being enough once the table is read + first -- a model the table keys exactly is answered there and never reaches + the OpenRouter tier, which is the point of reading it first. + """ + import litellm + + monkeypatch.setattr(litellm, "model_cost", {}) + _patch_litellm_info(monkeypatch, _litellm_miss) + + +def test_a_litellm_mapped_window_comes_from_litellm_with_no_network(monkeypatch): + _patch_litellm_info(monkeypatch, lambda m: {"max_input_tokens": 200000}) + counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + assert resolve_context_window("anthropic/claude-sonnet-4-5") == 200000 + assert counter["calls"] == 0 + + +def test_an_openrouter_window_falls_back_to_the_live_table(monkeypatch): + _patch_litellm_info(monkeypatch, _litellm_miss) + _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + assert resolve_context_window("openrouter/deepseek/deepseek-v4-pro") == 163840 + assert resolve_context_window("openrouter/deepseek-v4-pro") == 163840 + + +def test_a_direct_route_gets_no_window_from_the_gateways_table(monkeypatch): + """The window half of the hijack. A direct ``deepseek/`` route is not an + OpenRouter request, so OpenRouter's context length does not describe it.""" + _patch_litellm_blind(monkeypatch) + counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + assert resolve_context_window("deepseek/deepseek-v4-pro") is None + assert counter["calls"] == 0 + + +def test_the_snapshot_is_not_a_window_source(monkeypatch): + """A window sizes trimming, so it shapes the next request -- which is the line + the catalogue snapshot is deliberately kept on the other side of. It carries + labels and a price and no ``limit``, and this asserts it stays that way even + for a model it fully describes.""" + from raven.providers.catalog import model_cost + + assert model_cost("zai/glm-5.2"), "premise changed; pick another snapshot-only model" + _patch_litellm_blind(monkeypatch) + _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + assert resolve_context_window("zai/glm-5.2") is None + + +def test_a_window_unknown_to_every_source_is_none(monkeypatch): + _patch_litellm_info(monkeypatch, _litellm_miss) + _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + assert resolve_context_window("openrouter/some/model-not-listed") is None + + +# --- Models whose driver would start an interactive login --- + +_COPILOT_MODELS = [ + { + "id": "github_copilot/gpt-4.1", + "pricing": {"prompt": "0.000002", "completion": "0.000008"}, + "context_length": 128000, + } +] + + +def _forbid(recorder: list): + """A stub that records the call and misses. + + Recorded rather than raised: both lookups swallow exceptions to move to the + next candidate, so a probe that raises is caught and proves nothing. That is + how the first version of these tests passed against the unfixed code. + """ + + def _stub(*args, **kwargs): + recorder.append(kwargs.get("model") or (args[0] if args else "?")) + raise Exception("unmapped") + + return _stub + + +def test_the_window_of_a_login_prompting_model_comes_from_the_table(monkeypatch): + """Asking LiteLLM about a Copilot model starts a GitHub device flow. + + It resolves the model's credentials on the way to its metadata, so with no + token file on disk the lookup prints device codes to stdout and blocks -- + twice over, because the bare and the openrouter-prefixed candidate reach the + same driver. session.create runs this before the first turn, so the symptom + was a gateway that hung on opening. + """ + import litellm + + asked: list[str] = [] + monkeypatch.setattr(litellm, "get_model_info", _forbid(asked)) + counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + assert resolve_context_window("github_copilot/gpt-4.1") == 128000 + assert not asked, f"a login-prompting model was handed to LiteLLM: {asked}" + assert counter["calls"] == 0 + + +def test_the_rates_of_a_login_prompting_model_skip_litellm(monkeypatch): + """The same hang, on the path that runs after every single call. + + ``cost_per_token`` resolves credentials too, so the cost estimate blocked on + the same device flow. Fixing only the window lookup left this one, and + answering "can this be handed to LiteLLM" separately in each place is what + made the first attempt at this wrong. + """ + import litellm + + asked: list[str] = [] + monkeypatch.setattr(litellm, "cost_per_token", _forbid(asked)) + _patch_openrouter(monkeypatch, lambda req: _models_response(_COPILOT_MODELS)) + + token_rates("github_copilot/gpt-4.1", 1000, 100) + + assert not asked, f"a login-prompting model was handed to LiteLLM: {asked}" + + +def test_a_login_prompting_model_no_source_knows_is_never_asked(monkeypatch): + """No row and no safe way to ask: degrade, do not prompt. + + Both lookups fall through to what they already do for an unknown model -- the + caller keeps its configured window, and there are no rates. A read that runs + every turn is not worth a login prompt. + """ + import litellm + + asked: list[str] = [] + monkeypatch.setattr(litellm, "get_model_info", _forbid(asked)) + monkeypatch.setattr(litellm, "cost_per_token", _forbid(asked)) + _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + assert resolve_context_window("github_copilot/not-a-real-model") is None + assert token_rates("github_copilot/not-a-real-model", 1000, 100) is None + assert not asked, f"asked anyway: {asked}" + + +def test_reading_the_table_does_not_replace_asking_litellm(monkeypatch): + """The ask does more than key normalization, so it stays for everything else. + + "anthropic/claude-sonnet-4-5" is absent from the table -- it keys that model + bare -- and prefixed ids are the form Raven stores. Stripping the prefix to + read the table instead looked free and was not: for an openrouter-prefixed + candidate LiteLLM derives OpenRouter's own numbers, which are in no row, and + stripping answered three MiniMax models with the direct figure instead. + """ + import litellm + + assert "anthropic/claude-sonnet-4-5" not in getattr(litellm, "model_cost", {}), ( + "premise changed; this test proves nothing" + ) + _patch_litellm_info(monkeypatch, lambda m: {"max_input_tokens": 200000}) + counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + assert resolve_context_window("anthropic/claude-sonnet-4-5") == 200000 + assert counter["calls"] == 0 + + +def test_one_place_decides_whether_a_model_can_be_handed_to_litellm(): + """Both lookups reach the same authenticator, so both consult one answer. + + The first attempt at this guarded the metadata lookup only, and would have + needed the same decision again for the pricing call -- and again for + ``validate_environment``, which turned out to prompt as well. + """ + import ast + import pathlib + + source = (pathlib.Path(__file__).resolve().parents[1] / "raven" / "providers" / "rates.py").read_text() + tree = ast.parse(source) + owner = next(node for node in ast.walk(tree) if isinstance(node, ast.FunctionDef) and node.name == "_may_prompt") + allowed = range(owner.lineno, (owner.end_lineno or owner.lineno) + 1) + offenders = [ + f"line {i}: {line.strip()}" + for i, line in enumerate(source.splitlines(), 1) + if "authenticator.py" in line and i not in allowed + ] + assert not offenders, "ask _may_prompt instead:\n" + "\n".join(offenders) + + +# --- Field reading and id candidates --- + + +def test_prose_in_a_numeric_field_is_not_read_as_a_number(): + """LiteLLM ships a self-documenting row whose numeric fields hold sentences.""" + from raven.providers.rates import _numeric + + prose = {"max_input_tokens": "max input tokens, if the provider specifies it"} + assert _numeric(prose, "max_input_tokens") is None + assert _numeric({"max_input_tokens": 128000}, "max_input_tokens") == 128000 + assert _numeric({"max_tokens": 8192}, "max_input_tokens", "max_tokens") == 8192 + assert _numeric(None, "max_tokens") is None + + +def test_which_drivers_can_prompt_is_read_from_the_installed_litellm(): + """Derived, not snapshotted, so a LiteLLM bump cannot make it stale. + + A frozen list would need regenerating on every bump, and a stale one brings + the hang back for the vendor it missed. The driver ships ``authenticator.py`` + or it does not. + """ + from raven.providers.rates import _may_prompt + + assert _may_prompt("github_copilot/gpt-4.1") + assert _may_prompt("openrouter/github_copilot/gpt-4.1"), "any segment counts" + assert _may_prompt("chatgpt/gpt-5.1") + assert _may_prompt("gigachat/GigaChat-2-Max") + for safe in ("openai/gpt-4o", "anthropic/claude-sonnet-4-5", "deepseek/deepseek-v4-pro", "gpt-4o"): + assert not _may_prompt(safe), safe + + +def test_a_model_reached_by_region_or_subscription_is_looked_up_as_the_vendor_files_it(): + """Routing says where the request goes; the table is keyed by what the model + is. Asking with the routing id missed every time, so a Codex or MiniMax OAuth + model had no context window and no price at all.""" + assert rates._candidates("openai-codex/gpt-5.3-codex") == ["chatgpt/gpt-5.3-codex"] + assert rates._candidates("minimax-global/MiniMax-M3") == ["minimax/MiniMax-M3"] + assert rates._candidates("minimax-cn/MiniMax-M3") == ["minimax/MiniMax-M3"] + + # Everything else is asked as it routes, with the alias behind it. + assert rates._candidates("deepseek/deepseek-chat") == [ + "deepseek/deepseek-chat", + "openrouter/deepseek/deepseek-chat", + ] + assert rates._candidates("openrouter/anthropic/claude-opus-4.8") == ["openrouter/anthropic/claude-opus-4.8"] + + +# --- Plan billing --- + + +def test_plan_billing_is_declared_not_inferred_from_oauth(): + """OAuth is how you authenticate, not how you are charged -- Vertex is OAuth + and metered, so the flag cannot stand in for the other.""" + from raven.providers.registry import PROVIDERS + + plan_billed = {spec.name for spec in PROVIDERS if spec.billing == "plan"} + assert plan_billed == {"openai_codex", "github_copilot", "minimax_global", "minimax_cn"} + + +def test_a_plan_billed_provider_still_reports_a_window(): + """Occupancy is the measure that means something on a subscription, so the + window resolves even where no per-token figure describes the call.""" + for model in ("github_copilot/gpt-4o", "openai-codex/gpt-5.3-codex", "minimax-global/MiniMax-M3"): + assert resolve_context_window(model), f"{model}: window still expected" + + +def test_a_directly_routed_model_is_priced_as_the_vendor_prices_it(): + """The alias answers with OpenRouter's numbers, which a user routing straight + to the vendor does not pay. Asked alias-first, this model reported half its + window at half its price.""" + assert resolve_context_window("deepseek/deepseek-chat") == 131_072 + assert _rate_cost("deepseek/deepseek-chat", 1_000_000, 0) == pytest.approx(0.28) + + # Routed through the gateway, OpenRouter's own numbers are the right ones. + assert resolve_context_window("openrouter/deepseek/deepseek-chat") == 65_536 + + +def test_the_window_those_families_report_is_the_vendors_own(): + """Read from LiteLLM's table offline, so this is the number, not a default.""" + assert rates._try_litellm_context_window("openai-codex/gpt-5.3-codex") == 128_000 + assert rates._try_litellm_context_window("minimax-global/MiniMax-M3") == 1_000_000 + + +# --- Disk persistence of the OpenRouter catalog --- + +_DEEPSEEK_PRICE = (0.0000005, 0.0000015) + + +def _disk_payload(fetched_at, *, prompt="0.0000005", completion="0.0000015", version=None): + return { + "version": model_catalog_cache.CACHE_VERSION if version is None else version, + "fetched_at": fetched_at, + "models": { + "deepseek/deepseek-v4-pro": { + "pricing": {"prompt": prompt, "completion": completion}, + "context_length": 163840, + } + }, + } + + +@pytest.fixture +def disk_cache(tmp_path, monkeypatch): + """Point the OpenRouter disk cache at a temp file; never touch real ~/.raven.""" + path = tmp_path / "model-catalog.json" + monkeypatch.setattr(model_catalog_cache, "_CACHE_PATH", path, raising=False) + rates._OPENROUTER_CACHE.clear() + monkeypatch.setattr(rates, "_OPENROUTER_CACHE_TIME", 0.0) + return path + + +def test_cold_fetch_writes_disk_cache(monkeypatch, disk_cache): + """A cold network fetch persists the catalog as a versioned envelope.""" + _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + token_rates("openrouter/deepseek/deepseek-v4-pro", 1000, 500) + + assert disk_cache.exists() + payload = json.loads(disk_cache.read_text(encoding="utf-8")) + assert payload["version"] == model_catalog_cache.CACHE_VERSION + assert payload["fetched_at"] > 0 + assert "deepseek/deepseek-v4-pro" in payload["models"] + + +def test_warm_disk_hit_skips_network(monkeypatch, disk_cache): + """A fresh disk file hydrates the in-proc cache with zero network calls.""" + disk_cache.write_text(json.dumps(_disk_payload(time.time())), encoding="utf-8") + counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + cost = _rate_cost("openrouter/deepseek/deepseek-v4-pro", 1000, 500) + + assert counter["calls"] == 0 + assert cost == pytest.approx(1000 * _DEEPSEEK_PRICE[0] + 500 * _DEEPSEEK_PRICE[1], rel=1e-9) + + +def test_expired_disk_triggers_refetch(monkeypatch, disk_cache): + """A disk file older than the TTL is not served fresh -- the catalog refetches.""" + stale_at = time.time() - (rates._OPENROUTER_CACHE_TTL + 100) + disk_cache.write_text(json.dumps(_disk_payload(stale_at, prompt="9", completion="9")), encoding="utf-8") + counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + cost = _rate_cost("openrouter/deepseek/deepseek-v4-pro", 1000, 500) + + assert counter["calls"] == 1 + assert cost == pytest.approx(1000 * _DEEPSEEK_PRICE[0] + 500 * _DEEPSEEK_PRICE[1], rel=1e-9) + + +def test_version_mismatch_ignored(monkeypatch, disk_cache): + """A file whose version differs from CACHE_VERSION is treated as a miss.""" + disk_cache.write_text( + json.dumps(_disk_payload(time.time(), prompt="9", completion="9", version=999)), + encoding="utf-8", + ) + counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + cost = _rate_cost("openrouter/deepseek/deepseek-v4-pro", 1000, 500) + + assert counter["calls"] == 1 + assert cost == pytest.approx(1000 * _DEEPSEEK_PRICE[0] + 500 * _DEEPSEEK_PRICE[1], rel=1e-9) + # The bad-version file is overwritten with a current-version envelope. + assert json.loads(disk_cache.read_text(encoding="utf-8"))["version"] == model_catalog_cache.CACHE_VERSION + + +def test_corrupt_disk_degrades_to_network(monkeypatch, disk_cache): + """An unparseable cache file degrades to a miss and falls through to network.""" + disk_cache.write_text("{ this is not valid json", encoding="utf-8") + counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + cost = _rate_cost("openrouter/deepseek/deepseek-v4-pro", 1000, 500) + + assert counter["calls"] == 1 + assert cost is not None + # The corrupt file is replaced by a clean, parseable envelope. + assert json.loads(disk_cache.read_text(encoding="utf-8"))["version"] == model_catalog_cache.CACHE_VERSION + + +def test_network_fail_falls_back_to_stale_disk(monkeypatch, disk_cache): + """On a network failure with an empty in-proc cache, the stale disk file is served.""" + stale_at = time.time() - (rates._OPENROUTER_CACHE_TTL + 100) + disk_cache.write_text(json.dumps(_disk_payload(stale_at)), encoding="utf-8") + + def boom(req): + raise httpx.ConnectError("offline") + + _patch_openrouter(monkeypatch, boom) + + cost = _rate_cost("openrouter/deepseek/deepseek-v4-pro", 1000, 500) + + assert cost == pytest.approx(1000 * _DEEPSEEK_PRICE[0] + 500 * _DEEPSEEK_PRICE[1], rel=1e-9) + + +def test_disk_write_is_atomic(monkeypatch, disk_cache): + """The write leaves no temp file behind and the cache file parses cleanly.""" + _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + token_rates("openrouter/deepseek/deepseek-v4-pro", 1000, 500) + + assert list(disk_cache.parent.glob("*.tmp")) == [] + json.loads(disk_cache.read_text(encoding="utf-8")) diff --git a/tests/test_provider_resolution_invariants.py b/tests/test_provider_resolution_invariants.py index c628104c..54e4c06b 100644 --- a/tests/test_provider_resolution_invariants.py +++ b/tests/test_provider_resolution_invariants.py @@ -189,15 +189,38 @@ def spy(self: ProviderSpec, model: str) -> bool: assert ("anthropic", "anthropic/claude-opus-4-5") in observed +#: Everything a section can hold. Filling all of it is what makes the probe +#: independent: built from `providers.auth` instead, it asks the declaration what +#: this provider needs and then asserts routing agrees -- but routing asks the +#: same declaration, so the two agree by construction and a wrong declaration is +#: green. A section with every field set is configured under any declaration +#: this grammar can express, so "filled means routable" is a claim about routing. +_EVERY_CREDENTIAL_FIELD: dict[str, object] = { + "apiKey": "sk-probe", + "apiBase": "http://localhost:8000/v1", + "apiKeyList": ["sk-probe"], +} + + +def _fully_configured_section() -> dict[str, object]: + """A section holding every credential field, whatever this provider needs. + + Deliberately not derived from the provider's own declaration. An earlier + version read `auth_methods(spec)` to decide what to fill, which made the + probe and the thing it probes read the same source: declare that Anthropic + needs a field that does not exist and the section would grow that field, the + route would be found, and the test would pass. + """ + return dict(_EVERY_CREDENTIAL_FIELD) + + @pytest.mark.parametrize("spec", PROVIDERS, ids=lambda s: s.name) def test_a_configured_provider_answers_for_every_prefix_it_owns(spec: ProviderSpec) -> None: """Outcome check: whatever prefix a provider answers to must reach it.""" assert spec.route_names, f"{spec.name}: answers to no prefix at all" for prefix in spec.route_names: model = f"{prefix}/probe-model" - raw = {"apiKey": "sk-probe"} if not spec.is_oauth else {} - if spec.is_local: - raw = {"apiBase": "http://localhost:8000/v1"} + raw = _fully_configured_section() config = Config.model_validate({"providers": {spec.name: raw}}) _, matched = config._match_provider(model) assert matched == spec.name, f"{model!r} -> {matched}, registry says {spec.name}" @@ -267,6 +290,44 @@ def test_only_the_registry_reads_the_raw_via_driver_field() -> None: assert not offenders, "read spec.model_prefix instead of the raw field: " + ", ".join(offenders) +def test_the_wire_form_of_a_model_id_is_built_in_one_module() -> None: + """`providers.wire` owns the storage-form to wire-form conversion. + + The rule used to be spelled at each client, and the spellings drifted: the + standard path grew a canonicalizer for prefixes written in a former or + hyphenated spelling and the gateway path never did, so a local deployment + addressed as "hosted-vllm/..." came out double-prefixed. Collapsing it left + one place to fix that -- and it is fixed; this keeps a second place from + appearing. + + Matched on attribute access and on `getattr` by name, because the second is + how the wizard reads these today -- a bare identifier is not matched, so a + local named `model_prefix` (the head of a split id) does not trip it. + + The inbound family that used to sit in this list -- the code deciding what to + *store* rather than what to send -- has since been collapsed into the same + module, so only two readers remain and neither builds a wire id. + """ + owner = "raven/providers/wire.py" + allowed = { + owner, + # Decomposition, not construction: asks which upstream vendor a gateway + # id names, to look up what that vendor's model can do. + "raven/providers/litellm_provider.py", + # Strips a known prefix off a model id to recover the vendor's own id for + # a connectivity probe. Also decomposition. + "raven/cli/onboard_commands.py", + } + needles = (".model_prefix", ".skip_prefixes", '"model_prefix"', '"skip_prefixes"') + offenders = { + _rel(path) + for path in _production_files() + for line in path.read_text().splitlines() + if not line.lstrip().startswith("#") and any(n in line for n in needles) + } + assert offenders <= allowed, f"build the wire form in {owner}: {sorted(offenders - allowed)}" + + def test_no_module_outside_the_registry_splits_a_model_id_by_hand() -> None: """Prefix parsing lives in `split_model_id`. @@ -309,9 +370,12 @@ def test_find_by_keywords_is_imported_only_where_it_cannot_place_credentials() - the new caller does not place credentials. """ allowed = { - "raven/providers/litellm_provider.py", # caching + param quirks, after routing is settled - "raven/token_wise/cache_optimizer.py", - "raven/token_wise/system_and_tail_cache.py", + "raven/providers/litellm_provider.py", # param quirks, after routing is settled + # Asks which vendor's dialect a request speaks, never where its key goes. + # The two token_wise strategies used to be on this list with the same + # argument; they now ask this module instead of the registry, so the + # question is answered once rather than in three places. + "raven/providers/prompt_cache.py", "raven/config/schema.py", # asks only whether an id names a vendor at all } importers = { diff --git a/tests/test_provider_stream_fallback.py b/tests/test_provider_stream_fallback.py index 627bc6cf..3428acf5 100644 --- a/tests/test_provider_stream_fallback.py +++ b/tests/test_provider_stream_fallback.py @@ -7,10 +7,13 @@ from __future__ import annotations +import asyncio import json from typing import Any -from raven.providers.base import LLMProvider, LLMResponse, ToolCallRequest +import pytest + +from raven.providers.base import GenerationSettings, LLMProvider, LLMResponse, ToolCallRequest class _ChatOnlyProvider(LLMProvider): @@ -61,3 +64,188 @@ async def test_fallback_encodes_tool_calls_for_reconstruction() -> None: assert tc["id"] == "call_1" assert tc["function"]["name"] == "search" assert json.loads(tc["function"]["arguments"]) == {"q": "x"} + + +# --------------------------------------------------------------------------- +# The stream is closed on every exit, including the most likely one +# --------------------------------------------------------------------------- + + +class _FakeStream: + """A stream whose first pull can be made to hang, and that records aclose.""" + + def __init__(self, chunks, *, stall_first=False, refuse_first_pull=False, log=None, name="stream"): + self._chunks = list(chunks) + self._stall_first = stall_first + self._refuse_first_pull = refuse_first_pull + self._log = log + self._name = name + self.pulls = 0 + self.closed = False + + def __aiter__(self): + return self + + async def __anext__(self): + self.pulls += 1 + if self._stall_first and self.pulls == 1: + await asyncio.sleep(3600) + if self._refuse_first_pull and self.pulls == 1: + raise RuntimeError("litellm.BadRequestError: 400 cache_control: Extra inputs are not permitted") + if not self._chunks: + raise StopAsyncIteration + return self._chunks.pop(0) + + async def aclose(self): + if self._log is not None and not self.closed: + self._log.append(f"close:{self._name}") + self.closed = True + + +@pytest.mark.asyncio +async def test_a_first_chunk_timeout_still_closes_the_stream(monkeypatch): + """The likeliest timeout there is, and it used to leak. + + Hoisting the first pull out of the try/finally left the underlying HTTP + stream open on exactly the exit that happens most -- a gateway queueing or + cold-starting before the first byte. `aclosing()` upstream cannot recover it: + the exception has already driven the generator to a terminated state, where + `aclose()` is a no-op. + """ + from raven.providers.litellm_provider import LiteLLMProvider + + stream = _FakeStream([], stall_first=True) + provider = LiteLLMProvider(api_key="k", default_model="anthropic/claude-fable-5") + provider.generation = GenerationSettings(temperature=0, max_tokens=8, timeout=0.05) + monkeypatch.setattr( + "raven.providers.litellm_provider.acompletion", + _always(stream), + raising=False, + ) + + with pytest.raises(asyncio.TimeoutError): + async for _ in provider.chat_stream(messages=[{"role": "user", "content": "hi"}]): + pass + + assert stream.closed, "the stream was left open on a first-chunk timeout" + + +def _always(stream): + async def _acompletion(**kwargs): + return stream + + return _acompletion + + +class _RefusingOnce: + """An acompletion stand-in that refuses the field once, at a chosen point. + + Records the streams it hands out and the order of opens and closes, because + "the refused stream was closed before its replacement was opened" is a claim + about sequence, not just about a final state. + """ + + def __init__(self, *, at: str): + self.at = at + self.calls = 0 + self.streams: list[_FakeStream] = [] + self.events: list[str] = [] + + async def __call__(self, **kwargs): + self.calls += 1 + first_try = self.calls == 1 + self.events.append(f"open{self.calls}") + if self.at == "open" and first_try: + raise RuntimeError( + "litellm.BadRequestError: 400 messages.0.content.0.text.cache_control: Extra inputs are not permitted" + ) + stream = _FakeStream( + [_chunk()], + refuse_first_pull=self.at == "pull" and first_try, + log=self.events, + name=f"stream{self.calls}", + ) + self.streams.append(stream) + return stream + + +def _chunk(): + class _C: + choices: list = [] + + return _C() + + +@pytest.mark.parametrize("thrown_at", ["open", "pull"]) +@pytest.mark.asyncio +async def test_a_refused_stream_recovers_wherever_the_refusal_is_thrown(monkeypatch, thrown_at): + """Both points are "before the first chunk", and each route picks one. + + An OpenAI-shaped route raises while opening; a gateway that defers the + request until the first pull raises there. Covering only the pull would + leave the learned downgrade out of reach on the open-raising route -- and + that route is the one the TUI streams over. + """ + from raven.providers import prompt_cache + from raven.providers.litellm_provider import LiteLLMProvider + + prompt_cache.reset_suppressions() + acompletion = _RefusingOnce(at=thrown_at) + monkeypatch.setattr("raven.providers.litellm_provider.acompletion", acompletion, raising=False) + + model = "openrouter/anthropic/claude-3-haiku" + provider = LiteLLMProvider(api_key="k", default_model=model, provider_name="openrouter") + provider.generation = GenerationSettings(temperature=0, max_tokens=8, timeout=5) + + async for _ in provider.chat_stream(messages=[{"role": "user", "content": "hi"}], model=model): + pass + + assert acompletion.calls == 2, f"refusal at {thrown_at} was not retried" + assert prompt_cache.is_suppressed(model), f"refusal at {thrown_at} was not learned" + + # Every stream handed out is closed, and the refused one is closed *before* + # its replacement is opened -- at most one live at a time. Asserting only the + # final state left the close-before-reopen fix with nothing pinning it: + # deleting it kept every related test green. + assert all(s.closed for s in acompletion.streams), f"left open: {acompletion.events}" + if thrown_at == "pull": + assert acompletion.events.index("close:stream1") < acompletion.events.index("open2"), ( + f"the refused stream outlived its replacement's open: {acompletion.events}" + ) + prompt_cache.reset_suppressions() + + +@pytest.mark.asyncio +async def test_a_none_chunk_does_not_end_the_stream(monkeypatch): + """A chunk of None is a chunk, not the end of the stream. + + Pulling the first chunk before the loop needs a value meaning "there was + none"; reusing None for that let a provider yielding one truncate the + response silently, which is not what the loop did before it was restructured. + """ + from raven.providers.litellm_provider import LiteLLMProvider + + def _text(value): + class _Delta: + content = value + tool_calls = None + reasoning_content = None + + class _Choice: + delta = _Delta() + + class _C: + choices = [_Choice()] + usage = None + + return _C() + + stream = _FakeStream([_text("a"), None, _text("b")]) + provider = LiteLLMProvider(api_key="k", default_model="anthropic/claude-fable-5") + provider.generation = GenerationSettings(temperature=0, max_tokens=8, timeout=5) + monkeypatch.setattr("raven.providers.litellm_provider.acompletion", _always(stream), raising=False) + + seen = [d.content async for d in provider.chat_stream(messages=[{"role": "user", "content": "hi"}])] + + assert seen == ["a", "b"], f"the None chunk cut the stream short: {seen}" + assert stream.closed diff --git a/tests/test_provider_wire_model.py b/tests/test_provider_wire_model.py new file mode 100644 index 00000000..5e5246e4 --- /dev/null +++ b/tests/test_provider_wire_model.py @@ -0,0 +1,236 @@ +"""Characterization baseline for the storage-form to wire-form conversion. + +A stored model id is not what goes on the wire. That rule used to live in seven +places -- two branches of ``LiteLLMProvider._resolve_model``, Codex's own prefix +strip, Azure's URL builder, and three copies outside the providers package. It +now lives only in ``providers.wire``; this file is the baseline that made the +collapse safe, by pinning the wire output across it. + +Regenerated deliberately since, for fixes the storage contract required rather +than for drift. Eleven of the 149 entries moved when this landed, in three groups -- stated in full because "the wire output does +not move" is this file's whole claim, and a regeneration whose reasons cover only +part of the delta is that claim quietly weakened: + +* **Azure (3)** -- every stored id now names its provider, and that name has to + come off before the id becomes a URL path segment; +* **local deployments (4)** -- the same change made "hosted-vllm/x" and + "ollama/x" real stored ids, and the gateway branch double-prefixed them; +* **providers reached through another vendor's driver (4)** -- `custom/x` and + `siliconflow/x` used to go out as "openai/custom/x", carrying a segment the + upstream does not serve. Canonicalizing the gateway branch fixed this as a side + effect, which is why it was not in the first two reasons: it was found by + diffing against `main`, not by predicting it. + +So these tests are a snapshot, not a specification. They assert that today's +answer for every provider and every shape of model id is byte-for-byte what it +was before the refactor -- including the answers that are arguably wrong. A +defect found here gets reported, not fixed: a fix and a refactor in the same +step leave nothing to bisect against, which is how the previous attempt at this +came to be rolled back. + +The corpus is synthetic and derived from each spec's own fields rather than from +LiteLLM's catalogue, so a dependency bump cannot churn the baseline. + +Regenerate deliberately, never to make a red test green:: + + RAVEN_UPDATE_WIRE_BASELINE=1 uv run pytest tests/test_provider_wire_model.py +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +import pytest + +from raven.providers.registry import PROVIDERS, ProviderSpec, public_model_prefix + +BASELINE = Path(__file__).parent / "data" / "wire_model_baseline.json" + +#: Stand-in for a vendor's own model id. Deliberately free of any keyword that +#: the registry matches on, so a case only exercises the prefix rule under test. +PLAIN = "zz-probe-1" + + +def _corpus(spec: ProviderSpec) -> list[str]: + """Every shape of stored model id this provider has to answer for. + + Built from the spec's own fields so the set grows with the registry rather + than needing to be maintained beside it. + """ + ids = [ + PLAIN, + f"{spec.name}/{PLAIN}", + f"{public_model_prefix(spec)}/{PLAIN}", + # A model id whose vendor part itself contains a slash: "openai/gpt-oss" + # is Groq's own name for a model, and a gateway that strips one leading + # segment must not keep only the tail. + f"vendorx/{PLAIN}", + f"{spec.name}/vendorx/{PLAIN}", + ] + ids += [f"{alias}/{PLAIN}" for alias in spec.name_aliases] + if spec.via_driver: + ids.append(f"{spec.via_driver}/{PLAIN}") + ids += [f"{skip}{PLAIN}" for skip in spec.skip_prefixes] + if spec.default_model: + ids.append(spec.default_model) + if spec.keywords: + ids.append(f"{spec.keywords[0]}-probe") + # Deduplicate while keeping the order stable across runs. + return list(dict.fromkeys(ids)) + + +def _litellm_answers() -> dict[str, dict[str, str]]: + """What ``_resolve_model`` returns today, per provider, per stored id. + + The provider is built with an empty key on purpose: ``_setup_env`` is + skipped, so snapshotting cannot leak a credential into ``os.environ`` and + the answers stay independent of the machine running them. + """ + from raven.providers.litellm_provider import LiteLLMProvider + + out: dict[str, dict[str, str]] = {} + for spec in PROVIDERS: + provider = LiteLLMProvider(api_key="", default_model=PLAIN, provider_name=spec.name) + out[spec.name] = {model: provider._resolve_model(model) for model in _corpus(spec)} + return out + + +def _codex_answers() -> dict[str, str]: + from raven.providers.openai_codex_provider import _strip_model_prefix + + spec = next(s for s in PROVIDERS if s.name == "openai_codex") + extra = ["openai-codex/gpt-5.3-codex", "openai_codex/gpt-5.3-codex"] + return {model: _strip_model_prefix(model) for model in [*_corpus(spec), *extra]} + + +def _azure_answers() -> dict[str, str]: + """The URL Azure builds, which embeds the model id as a deployment name. + + Azure used to take the id verbatim, so a stored id carrying a prefix put + that prefix into the URL path. Once every stored id names its provider, that + is every Azure id -- so the provider's own name comes off here. The baseline + for this section was regenerated for that change; it is a fix the storage + contract required, not drift. + """ + from raven.providers.azure_openai_provider import AzureOpenAIProvider + + spec = next(s for s in PROVIDERS if s.name == "azure_openai") + provider = AzureOpenAIProvider(api_key="k", api_base="https://example.openai.azure.com") + return {model: provider._build_chat_url(model) for model in _corpus(spec)} + + +def _current() -> dict[str, Any]: + return { + "litellm_resolve_model": _litellm_answers(), + "codex_strip_model_prefix": _codex_answers(), + "azure_chat_url": _azure_answers(), + } + + +@pytest.fixture(scope="module") +def baseline() -> dict[str, Any]: + """The recorded answers. Regenerated only when explicitly asked. + + It used to regenerate whenever the file was missing, which made deleting it + a way to turn this suite green: the snapshot would be rewritten from + whatever the code currently does and then compared against itself. A + characterization test that can be satisfied by removing its own evidence + is not one. + """ + if os.environ.get("RAVEN_UPDATE_WIRE_BASELINE"): + BASELINE.parent.mkdir(parents=True, exist_ok=True) + BASELINE.write_text(json.dumps(_current(), indent=2, sort_keys=True) + "\n", encoding="utf-8") + + if not BASELINE.exists(): + pytest.fail( + f"{BASELINE.relative_to(Path(__file__).parents[1])} is missing. It is the record this " + "suite checks against, not an artifact it produces. Restore it from version control, or " + "regenerate deliberately with RAVEN_UPDATE_WIRE_BASELINE=1 and say why in the diff." + ) + return json.loads(BASELINE.read_text(encoding="utf-8")) + + +@pytest.mark.parametrize("spec", PROVIDERS, ids=lambda s: s.name) +def test_litellm_wire_form_is_unchanged(spec: ProviderSpec, baseline: dict[str, Any]) -> None: + """Every stored id this provider answers for still resolves to the same wire id.""" + recorded = baseline["litellm_resolve_model"].get(spec.name) + assert recorded is not None, f"no baseline for {spec.name}; regenerate deliberately" + + from raven.providers.litellm_provider import LiteLLMProvider + + provider = LiteLLMProvider(api_key="", default_model=PLAIN, provider_name=spec.name) + current = {model: provider._resolve_model(model) for model in _corpus(spec)} + assert current == recorded + + +def test_codex_wire_form_is_unchanged(baseline: dict[str, Any]) -> None: + assert _codex_answers() == baseline["codex_strip_model_prefix"] + + +def test_azure_wire_form_is_unchanged(baseline: dict[str, Any]) -> None: + assert _azure_answers() == baseline["azure_chat_url"] + + +def test_the_baseline_covers_every_registered_provider(baseline: dict[str, Any]) -> None: + """A provider added without a baseline entry would refactor unobserved.""" + missing = [s.name for s in PROVIDERS if s.name not in baseline["litellm_resolve_model"]] + assert not missing, f"regenerate the baseline for: {missing}" + + +def test_resolving_a_model_does_not_write_credentials_into_the_environment() -> None: + """The snapshot must not depend on -- or alter -- the machine it runs on. + + ``_setup_env`` exports the provider's key under its env var, and a snapshot + that tripped it would both leak and become order-dependent. + """ + before = dict(os.environ) + _litellm_answers() + assert dict(os.environ) == before + + +# --------------------------------------------------------------------------- +# Inbound: one spelling written, any spelling removes it +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("spec", PROVIDERS, ids=lambda s: s.name) +def test_both_write_paths_store_the_same_id(spec: ProviderSpec) -> None: + """The TUI and the wizard must write one model one way. + + They did not: for most providers the TUI stored a bare id while the wizard + stored a qualified one, so picking the same model in the two places put it in + the list twice. + """ + from raven.cli.onboard_commands import _format_model_for_provider + from raven.tui_rpc.methods.model import _stored_spelling + + assert _stored_spelling(spec.name, PLAIN) == _format_model_for_provider(spec.name, spec, PLAIN) + + +@pytest.mark.parametrize("spec", PROVIDERS, ids=lambda s: s.name) +def test_a_stored_id_always_names_its_provider(spec: ProviderSpec) -> None: + """Written bare, an id is claimed by keyword matching instead of by its owner.""" + from raven.providers.wire import stored_model_id + + stored = stored_model_id(spec.name, PLAIN) + assert "/" in stored, f"{spec.name}: stored {stored!r} with nothing naming the provider" + + +@pytest.mark.parametrize("spec", PROVIDERS, ids=lambda s: s.name) +def test_storing_an_already_qualified_id_is_idempotent(spec: ProviderSpec) -> None: + """Re-storing what was stored must not stack a second prefix.""" + from raven.providers.wire import stored_model_id + + once = stored_model_id(spec.name, PLAIN) + assert stored_model_id(spec.name, once) == once + + +@pytest.mark.parametrize("spec", PROVIDERS, ids=lambda s: s.name) +def test_a_bare_id_and_its_qualified_form_are_one_model(spec: ProviderSpec) -> None: + """Configs written before ids carried a provider still match the new form.""" + from raven.providers.wire import merge_key, stored_model_id + + assert merge_key(spec.name, PLAIN) == merge_key(spec.name, stored_model_id(spec.name, PLAIN)) diff --git a/tests/test_read_file_image.py b/tests/test_read_file_image.py index dd1749c6..6f50b8fd 100644 --- a/tests/test_read_file_image.py +++ b/tests/test_read_file_image.py @@ -21,7 +21,7 @@ from raven.agent.tools.filesystem import ReadFileTool from raven.agent.tools.registry import ToolRegistry from raven.providers.base import LLMProvider -from raven.token_wise import pricing as _pricing +from raven.providers import rates as _pricing # Captured before the autouse _no_openrouter_network fixture swaps it out. _REAL_FETCH = _pricing._fetch_openrouter_models @@ -1145,7 +1145,7 @@ def _catalog(monkeypatch, models: dict[str, list[str] | None]) -> None: times a day, so an assertion against it is an assertion about someone else's deploy. """ - from raven.token_wise import pricing + from raven.providers import rates as pricing built: dict[str, dict] = {} for model_id, mods in models.items(): @@ -1194,7 +1194,7 @@ def test_the_catalog_is_warmed_in_the_background_when_it_has_no_answer(monkeypat forever. Warmed off the request path because the fetch takes a 10s timeout. """ from raven.providers.capabilities import supports_vision - from raven.token_wise import pricing + from raven.providers import rates as pricing calls: list[str] = [] monkeypatch.setattr(pricing, "_OPENROUTER_CACHE", {}) @@ -1217,7 +1217,7 @@ def test_a_failed_warm_is_retried_once_the_cooldown_passes(monkeypatch) -> None: answering from an empty catalog for the rest of the process -- an attempt that failed says nothing about the next one.""" from raven.providers.capabilities import supports_vision - from raven.token_wise import pricing + from raven.providers import rates as pricing calls: list[str] = [] @@ -1251,7 +1251,7 @@ def test_the_warm_resolves_its_fetch_before_the_thread_starts(monkeypatch) -> No body that looked the fetch up on entry could therefore lose a race with whoever patched it -- a restored test seam would send a real request from inside the suite and write the real cache file.""" - from raven.token_wise import pricing + from raven.providers import rates as pricing calls: list[str] = [] captured: dict[str, object] = {} @@ -1283,7 +1283,7 @@ def start(self) -> None: def test_a_warm_that_cannot_reach_the_host_does_not_raise(monkeypatch) -> None: """The fetch degrades internally, but a thread that dies loudly writes a traceback into a user's terminal for a probe that has already answered.""" - from raven.token_wise import pricing + from raven.providers import rates as pricing def _boom() -> dict: raise RuntimeError("no route to host") @@ -1347,7 +1347,7 @@ def test_an_entry_written_before_modalities_were_kept_is_not_a_denial(monkeypatc def test_a_catalog_that_blows_up_does_not_break_the_probe(monkeypatch) -> None: """A capability probe must never be the thing that fails a turn.""" from raven.providers import capabilities - from raven.token_wise import pricing + from raven.providers import rates as pricing def _boom(model): raise RuntimeError("catalog on fire") @@ -1769,8 +1769,9 @@ def test_the_fetch_files_no_normalized_join_key_in_the_shared_table(monkeypatch) text-only verdict. Asserted through the real fetch: a hand-built table cannot see this, which is exactly why the mutation went unnoticed. """ + from raven.providers import model_catalog_cache + from raven.providers import rates as pricing from raven.providers.capabilities import supports_vision - from raven.token_wise import model_catalog_cache, pricing payload = { "data": [ @@ -1868,7 +1869,7 @@ def test_a_cold_verdict_is_not_cached_for_the_life_of_the_loop(monkeypatch) -> N background warm filling a table nothing re-reads -- so only a real verdict is remembered.""" from raven.agent.loop.main import AgentLoop - from raven.token_wise import pricing + from raven.providers import rates as pricing loop = object.__new__(AgentLoop) loop._vision_ok = {} diff --git a/tests/test_token_wise_cache_optimizer.py b/tests/test_token_wise_cache_optimizer.py index 96688c3d..9b4ad0cb 100644 --- a/tests/test_token_wise_cache_optimizer.py +++ b/tests/test_token_wise_cache_optimizer.py @@ -10,7 +10,7 @@ # Anthropic models support cache_control per the provider registry. ANTHROPIC_MODEL = "anthropic/claude-sonnet-4-5" -# A model that does NOT support prompt caching → strategy must be a no-op. +# A provider that does NOT accept `cache_control` blocks → strategy must be a no-op. NON_CACHE_MODEL = "deepseek/deepseek-chat" @@ -215,3 +215,53 @@ async def test_idempotent_repeated_application(): twice_m, _, _ = await opt.before_llm_call(once_m, None, ANTHROPIC_MODEL) # Same cache count; the marker is overwritten not duplicated. assert _count_breakpoints(once_m, None) == _count_breakpoints(twice_m, None) + + +def test_cache_control_follows_the_wire_format_not_the_model_catalogue(): + """`cache_control` is Anthropic-shaped; the question is who accepts one. + + LiteLLM's table carries a per-model `supports_prompt_caching`, and reading it + here looks like an upgrade: it says DeepSeek's and OpenAI's models cache, + which they do. But their caching is automatic and takes no breakpoints, so + acting on that flag stamped `cache_control` onto requests to APIs with + nowhere to put it -- and `ProviderSpec.supports_prompt_caching` already says + what it means, "this provider accepts the block". + + The lesson is the naming: two flags spelled the same answer different + questions, and the more precise-looking one was the wrong one. + """ + from pathlib import Path + + from raven.providers import capabilities + from raven.providers.registry import PROVIDERS + from raven.token_wise import cache_optimizer, system_and_tail_cache + + lying = [s.name for s in PROVIDERS if s.supports_prompt_caching and s.via_driver == "openai"] + assert not lying, f"marked as accepting cache_control while speaking OpenAI's API: {lying}" + + for module in (cache_optimizer, system_and_tail_cache): + source = Path(module.__file__).read_text(encoding="utf-8") + assert "capabilities import supports_prompt_caching" not in source, module.__name__ + assert not hasattr(capabilities, "supports_prompt_caching"), "the misfounded lookup is back" + + +def test_a_gateway_alone_does_not_decide_caching_for_what_it_fronts(): + """Both halves are required: the wire has somewhere to put the field, and the + model's vendor is the one that reads it. + + aihubmix / siliconflow / volcengine / custom speak an API with nowhere to + carry it, so nothing they front is marked. OpenRouter has somewhere to put + it and accepts it on every model it fronts -- and forwards it to vendors + that do not read it, which is what billed Gemini twice. So it is marked for + the Anthropic family and for nothing else. + """ + from raven.providers.litellm_provider import LiteLLMProvider + + for slug in ("siliconflow", "aihubmix", "volcengine", "custom"): + provider = LiteLLMProvider(api_key="", default_model="x", provider_name=slug) + assert provider._supports_cache_control("anthropic/claude-fable-5") is False, slug + + openrouter = LiteLLMProvider(api_key="", default_model="x", provider_name="openrouter") + assert openrouter._supports_cache_control("openrouter/anthropic/claude-fable-5") is True + for other in ("openrouter/google/gemini-3.5-flash", "openrouter/qwen/qwen3.7-max", "openrouter/anything"): + assert openrouter._supports_cache_control(other) is False, other diff --git a/tests/test_token_wise_pricing.py b/tests/test_token_wise_pricing.py index 01ca00f5..e485be5f 100644 --- a/tests/test_token_wise_pricing.py +++ b/tests/test_token_wise_pricing.py @@ -1,62 +1,23 @@ -"""Tests for raven.token_wise.pricing.""" +"""Tests for raven.token_wise.pricing -- the arithmetic on top of a rate. -from __future__ import annotations +Where a rate comes from is ``raven.providers.rates`` and is tested in +``test_provider_rates.py``. What is asserted here is what the cost formula does +with one: the cache multipliers, the plan-billed abstention, and degrading to +None rather than reporting a number nobody was charged. +""" -import json -import time +from __future__ import annotations -import httpx import pytest -from raven.token_wise import model_catalog_cache, pricing -from raven.token_wise.pricing import ( - _FALLBACK_PRICING, - estimate_cost_usd, - reset_warning_cache, - resolve_context_window, -) - -# The real fetch, captured before conftest's autouse guard stubs it to {}. -_REAL_FETCH = pricing._fetch_openrouter_models +from raven.token_wise.pricing import estimate_cost_usd, reset_warning_cache @pytest.fixture(autouse=True) def _reset_warning_state(): reset_warning_cache() - pricing._OPENROUTER_CACHE.clear() yield reset_warning_cache() - pricing._OPENROUTER_CACHE.clear() - - -def _patch_openrouter(monkeypatch, handler): - """Route pricing's real OpenRouter fetch through a MockTransport. - - Restores the real ``_fetch_openrouter_models`` (conftest stubs it to {} so - no test hits the network by default), then mocks the httpx transport. - Returns a counter dict whose ``["calls"]`` tracks network hits. - """ - counter = {"calls": 0} - - def counting_handler(request): - counter["calls"] += 1 - return handler(request) - - transport = httpx.MockTransport(counting_handler) - real_client = httpx.Client - - def client_factory(*args, **kwargs): - kwargs.setdefault("transport", transport) - return real_client(*args, **kwargs) - - monkeypatch.setattr(pricing, "_fetch_openrouter_models", _REAL_FETCH) - monkeypatch.setattr(pricing.httpx, "Client", client_factory) - monkeypatch.setattr(pricing, "_OPENROUTER_CACHE_TIME", 0.0) - return counter - - -def _models_response(models): - return httpx.Response(200, content=json.dumps({"data": models})) def test_known_anthropic_model_returns_positive_cost(): @@ -67,20 +28,8 @@ def test_known_anthropic_model_returns_positive_cost(): def test_unknown_model_returns_none(): - """Models LiteLLM doesn't know about and we don't have fallback for → None.""" - cost = estimate_cost_usd("nonexistent-vendor/imaginary-model-9000", 100, 100) - assert cost is None - - -def test_fallback_pricing_used_when_litellm_misses(): - """A model in our manual table should yield a finite cost even if LiteLLM lacks it.""" - model = next(iter(_FALLBACK_PRICING)) - p_rate, c_rate = _FALLBACK_PRICING[model] - cost = estimate_cost_usd(model, 1000, 500) - assert cost is not None - # Should equal the fallback exactly (or be at least that much, if LiteLLM also has it). - expected = 1000 * p_rate + 500 * c_rate - assert cost == pytest.approx(expected, rel=0.01) + """A model no rate tier knows costs nothing reportable, not zero.""" + assert estimate_cost_usd("nonexistent-vendor/imaginary-model-9000", 100, 100) is None def test_cache_read_is_cheaper_than_fresh_input(): @@ -88,7 +37,6 @@ def test_cache_read_is_cheaper_than_fresh_input(): base = estimate_cost_usd("anthropic/claude-sonnet-4-5", 1000, 0) cached = estimate_cost_usd("anthropic/claude-sonnet-4-5", 0, 0, cache_read_tokens=1000) assert base is not None and cached is not None - # Cache read is 10% of base prompt rate. assert cached == pytest.approx(base * 0.1, rel=0.01) @@ -101,25 +49,7 @@ def test_cache_write_more_expensive_than_fresh_input(): def test_zero_tokens_returns_zero_cost(): - cost = estimate_cost_usd("anthropic/claude-sonnet-4-5", 0, 0) - assert cost == 0.0 - - -def test_unknown_model_warns_only_once(caplog): - """Repeated estimates for the same unknown model must not flood the log.""" - import loguru - - seen: list[str] = [] - handler_id = loguru.logger.add(lambda m: seen.append(m), level="WARNING") - try: - estimate_cost_usd("ghost-vendor/never-heard-of", 10, 10) - estimate_cost_usd("ghost-vendor/never-heard-of", 10, 10) - estimate_cost_usd("ghost-vendor/never-heard-of", 10, 10) - finally: - loguru.logger.remove(handler_id) - - matching = [m for m in seen if "ghost-vendor/never-heard-of" in m] - assert len(matching) == 1, f"Expected 1 warning, got {len(matching)}: {matching}" + assert estimate_cost_usd("anthropic/claude-sonnet-4-5", 0, 0) == 0.0 def test_combined_input_output_and_cache(): @@ -135,486 +65,34 @@ def test_combined_input_output_and_cache(): cache_write_tokens=1000, ) assert full is not None - expected = base + out + base * 0.1 + base * 1.25 - assert full == pytest.approx(expected, rel=0.01) - - -_DEEPSEEK_MODELS = [ - { - "id": "deepseek/deepseek-v4-pro", - "context_length": 163840, - "pricing": {"prompt": "0.0000005", "completion": "0.0000015"}, - } -] - - -def test_openrouter_unmapped_model_yields_live_cost(monkeypatch): - """A model LiteLLM doesn't map gets a non-zero cost from OpenRouter's API.""" - _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - cost = estimate_cost_usd("openrouter/deepseek/deepseek-v4-pro", 1000, 500) - - assert cost is not None - assert cost == pytest.approx(1000 * 0.0000005 + 500 * 0.0000015, rel=1e-9) - - -def test_openrouter_bare_alias_lookup(monkeypatch): - """The bare model name (no vendor prefix) resolves via the double-keyed cache.""" - _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - cost = estimate_cost_usd("openrouter/deepseek-v4-pro", 1000, 0) - - assert cost == pytest.approx(1000 * 0.0000005, rel=1e-9) - - -def test_openrouter_miss_degrades_to_none(monkeypatch): - """An OpenRouter model absent from the /models table still degrades to None.""" - _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - cost = estimate_cost_usd("openrouter/some/model-not-listed", 1000, 500) - - assert cost is None - - -def test_openrouter_offline_degrades_to_none(monkeypatch): - """A network failure must never fabricate a rate — cost falls to None.""" - - def boom(req): - raise httpx.ConnectError("offline") - - _patch_openrouter(monkeypatch, boom) - - cost = estimate_cost_usd("openrouter/deepseek/deepseek-v4-pro", 1000, 500) - - assert cost is None - - -def test_openrouter_response_cached_for_an_hour(monkeypatch): - """The /models table is fetched once and reused across estimates.""" - counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - estimate_cost_usd("openrouter/deepseek/deepseek-v4-pro", 1000, 500) - estimate_cost_usd("openrouter/deepseek/deepseek-v4-pro", 10, 20) - - assert counter["calls"] == 1 - - -def test_non_openrouter_unmapped_model_consults_catalog(monkeypatch): - """Tier 2: any LiteLLM-miss model (not just openrouter/) consults the catalog, - and degrades to None when absent.""" - counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - cost = estimate_cost_usd("nonexistent-vendor/imaginary-model-9000", 100, 100) - - assert cost is None - assert counter["calls"] == 1 - - -_UNMAPPED_CATALOG = [ - { - "id": "fakevendor/imaginary-priced-9000", - "context_length": 163840, - "pricing": {"prompt": "0.0000005", "completion": "0.0000015"}, - } -] - - -def test_non_openrouter_model_priced_via_catalog(monkeypatch): - """Tier 2: a bare provider model LiteLLM misses is priced off the OpenRouter - catalog, no openrouter/ prefix required. Uses a vendor LiteLLM does not know - so Tier 1 genuinely misses and the catalog path is exercised.""" - _patch_openrouter(monkeypatch, lambda req: _models_response(_UNMAPPED_CATALOG)) - - cost = estimate_cost_usd("fakevendor/imaginary-priced-9000", 1000, 500) - - assert cost == pytest.approx(1000 * 0.0000005 + 500 * 0.0000015, rel=1e-9) - - -def _patch_litellm_info(monkeypatch, fn): - """Stub litellm.get_model_info (offline) — fn(model) returns a dict or raises.""" - import litellm - - monkeypatch.setattr(litellm, "get_model_info", fn) - - -def _litellm_miss(_model): - raise Exception("This model isn't mapped yet") - - -def _patch_litellm_blind(monkeypatch): - """Make LiteLLM miss for real: the price table is consulted before the ask. - - Stubbing ``get_model_info`` alone stopped being enough once the table is read - first -- a model the table keys exactly is answered there and never reaches - the OpenRouter tier, which is the point of reading it first. - """ - import litellm - - monkeypatch.setattr(litellm, "model_cost", {}) - _patch_litellm_info(monkeypatch, _litellm_miss) - - -def test_resolve_context_window_from_litellm_no_network(monkeypatch): - """Tier 1: a LiteLLM-mapped model's window comes from LiteLLM, no OpenRouter hit.""" - _patch_litellm_info(monkeypatch, lambda m: {"max_input_tokens": 200000}) - counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - assert resolve_context_window("anthropic/claude-sonnet-4-5") == 200000 - assert counter["calls"] == 0 - - -def test_resolve_context_window_from_openrouter_when_litellm_misses(monkeypatch): - """An OpenRouter model LiteLLM lags on falls back to the live /models table.""" - _patch_litellm_info(monkeypatch, _litellm_miss) - _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - assert resolve_context_window("openrouter/deepseek/deepseek-v4-pro") == 163840 - assert resolve_context_window("openrouter/deepseek-v4-pro") == 163840 - - -def test_resolve_context_window_non_openrouter_via_catalog(monkeypatch): - """Tier 2: a bare provider model LiteLLM misses resolves via the OpenRouter catalog.""" - _patch_litellm_blind(monkeypatch) - _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - assert resolve_context_window("deepseek/deepseek-v4-pro") == 163840 - - -def test_resolve_context_window_unknown_returns_none(monkeypatch): - """Unknown to both LiteLLM and the OpenRouter catalog resolves to None.""" - _patch_litellm_info(monkeypatch, _litellm_miss) - _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - assert resolve_context_window("openrouter/some/model-not-listed") is None - - -_COPILOT_MODELS = [ - { - "id": "github_copilot/gpt-4.1", - "pricing": {"prompt": "0.000002", "completion": "0.000008"}, - "context_length": 128000, - } -] - - -def _forbid(recorder: list): - """A stub that records the call and misses. - - Recorded rather than raised: both lookups swallow exceptions to move to the - next candidate, so a probe that raises is caught and proves nothing. That is - how the first version of these tests passed against the unfixed code. - """ - - def _stub(*args, **kwargs): - recorder.append(kwargs.get("model") or (args[0] if args else "?")) - raise Exception("unmapped") - - return _stub + assert full == pytest.approx(base + out + base * 0.1 + base * 1.25, rel=0.01) -def test_the_window_of_a_login_prompting_model_comes_from_the_table(monkeypatch): - """Asking LiteLLM about a Copilot model starts a GitHub device flow. - - It resolves the model's credentials on the way to its metadata, so with no - token file on disk one lookup printed six device codes to stdout and blocked - for 410 seconds -- two three-attempt login cycles, because the bare and the - openrouter-prefixed candidate reach the same driver. session.create runs this - before the first turn, so the symptom was a gateway that hung on opening. - """ - import litellm - - asked: list[str] = [] - monkeypatch.setattr(litellm, "get_model_info", _forbid(asked)) - counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - assert resolve_context_window("github_copilot/gpt-4.1") == 128000 - assert not asked, f"a login-prompting model was handed to LiteLLM: {asked}" - assert counter["calls"] == 0 - - -def test_the_rates_of_a_login_prompting_model_skip_litellm(monkeypatch): - """The same hang, on the path that runs after every single call. - - ``cost_per_token`` resolves credentials too, so the cost estimate blocked on - the same device flow. Fixing only the window lookup left this one, and - answering "can this be handed to LiteLLM" separately in each place is what - made the first attempt at this wrong. - - The estimate is absent rather than wrong: this provider is billed by seat, so - the live catalogue's per-token rate is not what the user pays. What this test - holds either way is that LiteLLM is never the one asked. - """ - import litellm - - asked: list[str] = [] - monkeypatch.setattr(litellm, "cost_per_token", _forbid(asked)) - _patch_openrouter(monkeypatch, lambda req: _models_response(_COPILOT_MODELS)) - - assert estimate_cost_usd("github_copilot/gpt-4.1", 1000, 100) is None - assert not asked, f"a login-prompting model was handed to LiteLLM: {asked}" - - -def test_a_login_prompting_model_the_table_does_not_price_is_never_asked(monkeypatch): - """No row and no safe way to ask: degrade, do not prompt. - - Both lookups fall through to what they already do for an unknown model -- the - caller keeps its configured window, and the cost estimate is None. A read that - runs every turn is not worth a login prompt. - """ - import litellm - - asked: list[str] = [] - monkeypatch.setattr(litellm, "get_model_info", _forbid(asked)) - monkeypatch.setattr(litellm, "cost_per_token", _forbid(asked)) - _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - assert resolve_context_window("github_copilot/not-a-real-model") is None - assert estimate_cost_usd("github_copilot/not-a-real-model", 1000, 100) is None - assert not asked, f"asked anyway: {asked}" - - -def test_reading_the_table_does_not_replace_asking_litellm(monkeypatch): - """The ask does more than key normalization, so it stays for everything else. - - "anthropic/claude-sonnet-4-5" is absent from the table -- it keys that model - bare -- and prefixed ids are the form Raven stores. Stripping the prefix to - read the table instead looked free and was not: for an openrouter-prefixed - candidate LiteLLM derives OpenRouter's own numbers, which are in no row, and - stripping answered three MiniMax models with the direct figure instead. - """ - import litellm - - assert "anthropic/claude-sonnet-4-5" not in getattr(litellm, "model_cost", {}), ( - "premise changed; this test proves nothing" - ) - _patch_litellm_info(monkeypatch, lambda m: {"max_input_tokens": 200000}) - counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - assert resolve_context_window("anthropic/claude-sonnet-4-5") == 200000 - assert counter["calls"] == 0 - - -def test_prose_in_a_numeric_field_is_not_read_as_a_number(): - """LiteLLM ships a self-documenting row whose numeric fields hold sentences.""" - from raven.token_wise.pricing import _numeric - - prose = {"max_input_tokens": "max input tokens, if the provider specifies it"} - assert _numeric(prose, "max_input_tokens") is None - assert _numeric({"max_input_tokens": 128000}, "max_input_tokens") == 128000 - assert _numeric({"max_tokens": 8192}, "max_input_tokens", "max_tokens") == 8192 - assert _numeric(None, "max_tokens") is None - - -def test_which_drivers_can_prompt_is_read_from_the_installed_litellm(): - """Derived, not snapshotted, so a LiteLLM bump cannot make it stale. - - A frozen list would need regenerating on every bump, and a stale one brings - the hang back for the vendor it missed. The driver ships ``authenticator.py`` - or it does not. - """ - from raven.token_wise.pricing import _may_prompt - - assert _may_prompt("github_copilot/gpt-4.1") - assert _may_prompt("openrouter/github_copilot/gpt-4.1"), "any segment counts" - assert _may_prompt("chatgpt/gpt-5.1") - assert _may_prompt("gigachat/GigaChat-2-Max") - for safe in ("openai/gpt-4o", "anthropic/claude-sonnet-4-5", "deepseek/deepseek-v4-pro", "gpt-4o"): - assert not _may_prompt(safe), safe - +def test_unknown_model_warns_only_once(): + """Repeated estimates for the same unknown model must not flood the log.""" + import loguru -def test_a_model_reached_by_region_or_subscription_is_looked_up_as_the_vendor_files_it(): - """Routing says where the request goes; the table is keyed by what the model - is. Asking with the routing id missed every time, so a Codex or MiniMax OAuth - model had no context window and no price at all.""" - assert pricing._candidates("openai-codex/gpt-5.3-codex") == ["chatgpt/gpt-5.3-codex"] - assert pricing._candidates("minimax-global/MiniMax-M3") == ["minimax/MiniMax-M3"] - assert pricing._candidates("minimax-cn/MiniMax-M3") == ["minimax/MiniMax-M3"] + seen: list[str] = [] + handler_id = loguru.logger.add(lambda m: seen.append(m), level="WARNING") + try: + for _ in range(3): + estimate_cost_usd("ghost-vendor/never-heard-of", 10, 10) + finally: + loguru.logger.remove(handler_id) - # Everything else is asked as it routes, with the alias behind it. - assert pricing._candidates("deepseek/deepseek-chat") == [ - "deepseek/deepseek-chat", - "openrouter/deepseek/deepseek-chat", - ] - assert pricing._candidates("openrouter/anthropic/claude-opus-4.8") == ["openrouter/anthropic/claude-opus-4.8"] + matching = [m for m in seen if "ghost-vendor/never-heard-of" in m] + assert len(matching) == 1, f"Expected 1 warning, got {len(matching)}: {matching}" def test_a_plan_billed_provider_reports_no_per_token_cost(): """The subscription is the price, so no per-token figure describes the call. - LiteLLM files these models at zero, which the tiers below read as "unknown" - and answered with the pay-as-you-go rate the user is not paying: $2.50 per - million tokens for a Copilot seat. Windows still resolve -- occupancy is the - measure that means something on a plan. + LiteLLM files these models at zero, which the rate ladder reads as "unknown" + and would answer with the pay-as-you-go rate the user is not paying: $2.50 per + million tokens for a Copilot seat. """ for model in ("github_copilot/gpt-4o", "openai-codex/gpt-5.3-codex", "minimax-global/MiniMax-M3"): assert estimate_cost_usd(model, 1_000_000, 0) is None, model - assert pricing.resolve_context_window(model), f"{model}: window still expected" # The same vendor's metered API is unaffected: that one is per-token. assert estimate_cost_usd("minimax/MiniMax-M3", 1_000_000, 0) == pytest.approx(0.3) - - -def test_plan_billing_is_declared_not_inferred_from_oauth(): - """OAuth is how you authenticate, not how you are charged -- Vertex is OAuth - and metered, so the flag cannot stand in for the other.""" - from raven.providers.registry import PROVIDERS - - plan_billed = {spec.name for spec in PROVIDERS if spec.billing == "plan"} - assert plan_billed == {"openai_codex", "github_copilot", "minimax_global", "minimax_cn"} - - -def test_a_directly_routed_model_is_priced_as_the_vendor_prices_it(): - """The alias answers with OpenRouter's numbers, which a user routing straight - to the vendor does not pay. Asked alias-first, this model reported half its - window at half its price.""" - assert pricing.resolve_context_window("deepseek/deepseek-chat") == 131_072 - assert pricing.estimate_cost_usd("deepseek/deepseek-chat", 1_000_000, 0) == pytest.approx(0.28) - - # Routed through the gateway, OpenRouter's own numbers are the right ones. - assert pricing.resolve_context_window("openrouter/deepseek/deepseek-chat") == 65_536 - - -def test_the_window_those_families_report_is_the_vendors_own(): - """Read from LiteLLM's table offline, so this is the number, not a default.""" - assert pricing._try_litellm_context_window("openai-codex/gpt-5.3-codex") == 128_000 - assert pricing._try_litellm_context_window("minimax-global/MiniMax-M3") == 1_000_000 - - -def test_one_place_decides_whether_a_model_can_be_handed_to_litellm(): - """Both lookups reach the same authenticator, so both consult one answer. - - The first attempt at this guarded the metadata lookup only, and would have - needed the same decision again for the pricing call -- and again for - ``validate_environment``, which turned out to prompt as well. - """ - import ast - import pathlib - - source = (pathlib.Path(__file__).resolve().parents[1] / "raven" / "token_wise" / "pricing.py").read_text() - tree = ast.parse(source) - owner = next(node for node in ast.walk(tree) if isinstance(node, ast.FunctionDef) and node.name == "_may_prompt") - allowed = range(owner.lineno, (owner.end_lineno or owner.lineno) + 1) - offenders = [ - f"line {i}: {line.strip()}" - for i, line in enumerate(source.splitlines(), 1) - if "authenticator.py" in line and i not in allowed - ] - assert not offenders, "ask _may_prompt instead:\n" + "\n".join(offenders) - - -# --- Disk persistence of the OpenRouter catalog --- - -_DEEPSEEK_PRICE = (0.0000005, 0.0000015) - - -def _disk_payload(fetched_at, *, prompt="0.0000005", completion="0.0000015", version=None): - return { - "version": model_catalog_cache.CACHE_VERSION if version is None else version, - "fetched_at": fetched_at, - "models": { - "deepseek/deepseek-v4-pro": { - "pricing": {"prompt": prompt, "completion": completion}, - "context_length": 163840, - } - }, - } - - -@pytest.fixture -def disk_cache(tmp_path, monkeypatch): - """Point the OpenRouter disk cache at a temp file; never touch real ~/.raven.""" - path = tmp_path / "model-catalog.json" - monkeypatch.setattr(model_catalog_cache, "_CACHE_PATH", path, raising=False) - pricing._OPENROUTER_CACHE.clear() - monkeypatch.setattr(pricing, "_OPENROUTER_CACHE_TIME", 0.0) - return path - - -def test_cold_fetch_writes_disk_cache(monkeypatch, disk_cache): - """A cold network fetch persists the catalog as a versioned envelope.""" - _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - estimate_cost_usd("openrouter/deepseek/deepseek-v4-pro", 1000, 500) - - assert disk_cache.exists() - payload = json.loads(disk_cache.read_text(encoding="utf-8")) - assert payload["version"] == model_catalog_cache.CACHE_VERSION - assert payload["fetched_at"] > 0 - assert "deepseek/deepseek-v4-pro" in payload["models"] - - -def test_warm_disk_hit_skips_network(monkeypatch, disk_cache): - """A fresh disk file hydrates the in-proc cache with zero network calls.""" - disk_cache.write_text(json.dumps(_disk_payload(time.time())), encoding="utf-8") - counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - cost = estimate_cost_usd("openrouter/deepseek/deepseek-v4-pro", 1000, 500) - - assert counter["calls"] == 0 - assert cost == pytest.approx(1000 * _DEEPSEEK_PRICE[0] + 500 * _DEEPSEEK_PRICE[1], rel=1e-9) - - -def test_expired_disk_triggers_refetch(monkeypatch, disk_cache): - """A disk file older than the TTL is not served fresh — the catalog refetches.""" - stale_at = time.time() - (pricing._OPENROUTER_CACHE_TTL + 100) - disk_cache.write_text(json.dumps(_disk_payload(stale_at, prompt="9", completion="9")), encoding="utf-8") - counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - cost = estimate_cost_usd("openrouter/deepseek/deepseek-v4-pro", 1000, 500) - - assert counter["calls"] == 1 - assert cost == pytest.approx(1000 * _DEEPSEEK_PRICE[0] + 500 * _DEEPSEEK_PRICE[1], rel=1e-9) - - -def test_version_mismatch_ignored(monkeypatch, disk_cache): - """A file whose version differs from CACHE_VERSION is treated as a miss.""" - disk_cache.write_text( - json.dumps(_disk_payload(time.time(), prompt="9", completion="9", version=999)), - encoding="utf-8", - ) - counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - cost = estimate_cost_usd("openrouter/deepseek/deepseek-v4-pro", 1000, 500) - - assert counter["calls"] == 1 - assert cost == pytest.approx(1000 * _DEEPSEEK_PRICE[0] + 500 * _DEEPSEEK_PRICE[1], rel=1e-9) - # The bad-version file is overwritten with a current-version envelope. - assert json.loads(disk_cache.read_text(encoding="utf-8"))["version"] == model_catalog_cache.CACHE_VERSION - - -def test_corrupt_disk_degrades_to_network(monkeypatch, disk_cache): - """An unparseable cache file degrades to a miss and falls through to network.""" - disk_cache.write_text("{ this is not valid json", encoding="utf-8") - counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - cost = estimate_cost_usd("openrouter/deepseek/deepseek-v4-pro", 1000, 500) - - assert counter["calls"] == 1 - assert cost is not None - # The corrupt file is replaced by a clean, parseable envelope. - assert json.loads(disk_cache.read_text(encoding="utf-8"))["version"] == model_catalog_cache.CACHE_VERSION - - -def test_network_fail_falls_back_to_stale_disk(monkeypatch, disk_cache): - """On a network failure with an empty in-proc cache, the stale disk file is served.""" - stale_at = time.time() - (pricing._OPENROUTER_CACHE_TTL + 100) - disk_cache.write_text(json.dumps(_disk_payload(stale_at)), encoding="utf-8") - - def boom(req): - raise httpx.ConnectError("offline") - - _patch_openrouter(monkeypatch, boom) - - cost = estimate_cost_usd("openrouter/deepseek/deepseek-v4-pro", 1000, 500) - - assert cost == pytest.approx(1000 * _DEEPSEEK_PRICE[0] + 500 * _DEEPSEEK_PRICE[1], rel=1e-9) - - -def test_disk_write_is_atomic(monkeypatch, disk_cache): - """The write leaves no temp file behind and the cache file parses cleanly.""" - _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) - - estimate_cost_usd("openrouter/deepseek/deepseek-v4-pro", 1000, 500) - - assert list(disk_cache.parent.glob("*.tmp")) == [] - json.loads(disk_cache.read_text(encoding="utf-8")) diff --git a/tests/test_tui_rpc_config.py b/tests/test_tui_rpc_config.py index 66e031a7..523ee1a8 100644 --- a/tests/test_tui_rpc_config.py +++ b/tests/test_tui_rpc_config.py @@ -197,6 +197,11 @@ async def test_config_set_model_reassigns_loop_and_persists(fake_home: Path, mon async def test_config_set_model_bare_derives_provider(fake_home: Path) -> None: # A bare `/model ` carries no provider; _set_model must derive it from # the model so a previously-forced provider does not silently mis-route. + # Configured, because an id naming a vendor with no section is deliberately + # left on `auto`: pinning it would stop routing before the fallback that lets + # a gateway serve that vendor's models. + _pin(fake_home, "auto", {"anthropic": {"api_key": "sk-ant"}}) + result = await config_set( {"key": "model", "value": "anthropic/claude-opus-4-8"}, agent_loop_factory=lambda: None, @@ -207,6 +212,23 @@ async def test_config_set_model_bare_derives_provider(fake_home: Path) -> None: assert cfg["agents"]["defaults"]["provider"] == "anthropic" +async def test_config_set_model_leaves_an_unconfigured_vendor_on_auto(fake_home: Path) -> None: + """The picker must not write a pin that stops routing. + + A pin is answered with the named vendor's section whether or not it holds + credentials, so pinning an unconfigured one fails every request on a missing + key -- never reaching the fallback written for a gateway serving that + vendor's models. An OpenRouter-only install picking `anthropic/...` would + reach nothing at all. + """ + _pin(fake_home, "auto", {"openrouter": {"api_key": "sk-or"}}) + + await config_set({"key": "model", "value": "anthropic/claude-opus-4-8"}, agent_loop_factory=lambda: None) + + cfg = json.loads((fake_home / ".raven" / "config.json").read_text()) + assert cfg["agents"]["defaults"]["provider"] == "auto" + + async def test_config_set_model_rejected_during_active_turn(fake_home: Path, monkeypatch) -> None: import raven.tui_rpc.methods.config as config_mod @@ -377,7 +399,10 @@ async def test_a_bare_id_the_pinned_provider_does_serve_keeps_the_pin(fake_home: assert result["applied"] is True cfg = json.loads((fake_home / ".raven" / "config.json").read_text()) assert cfg["agents"]["defaults"]["provider"] == "mistral" - assert cfg["agents"]["defaults"]["model"] == "mistral-large-latest" + # Stored naming its provider, which is what every surface writes: the same + # input through `raven provider use` produced the qualified form while this + # one kept it bare, so the two disagreed about what the user had chosen. + assert cfg["agents"]["defaults"]["model"] == "mistral/mistral-large-latest" async def test_a_local_deployment_keeps_its_pin_for_any_bare_id(fake_home: Path) -> None: diff --git a/tests/test_tui_rpc_model.py b/tests/test_tui_rpc_model.py index e3f5c825..0e689c76 100644 --- a/tests/test_tui_rpc_model.py +++ b/tests/test_tui_rpc_model.py @@ -78,8 +78,18 @@ async def test_options_authed_provider_lists_models(fake_home: Path) -> None: # catalogue (deduped). The order is the contract: recommendations stay at the # top of a list the catalogue makes long. assert entry["models"][:2] == ["claude-opus-4-8", "claude-sonnet-4-5"] - curated = common_models_for("anthropic") + + # The configured entries are written in the pre-contract bare spelling, and + # the curated list carries the qualified one. They are the same two models, + # so the shortlist contributes everything except those -- listing a model the + # user already has, under the other spelling, is the duplicate the picker + # used to show. + from raven.providers.wire import merge_key + + configured_keys = {merge_key("anthropic", m) for m in ("claude-opus-4-8", "claude-sonnet-4-5")} + curated = [m for m in common_models_for("anthropic") if merge_key("anthropic", m) not in configured_keys] assert entry["models"][2 : 2 + len(curated)] == curated + assert len(entry["models"]) == len({merge_key("anthropic", m) for m in entry["models"]}), "a model is listed twice" assert entry["total_models"] > 2 + len(curated), "the catalogue tier added nothing" assert entry["auth_type"] == "key" assert entry["key_env"] == "ANTHROPIC_API_KEY" @@ -204,10 +214,11 @@ async def test_disconnect_clears_creds(fake_home: Path) -> None: async def test_add_model_reflected_in_options(fake_home: Path) -> None: await model_save_key({"slug": "anthropic", "api_key": "sk-ant-xxx"}) result = await model_add_model({"slug": "anthropic", "model": "claude-opus-4-8"}) - assert "claude-opus-4-8" in result["provider"]["models"] + # Stored qualified: a bare id is claimed by keyword matching instead. + assert "anthropic/claude-opus-4-8" in result["provider"]["models"] options = await model_options({}) - assert "claude-opus-4-8" in _entry(options, "anthropic")["models"] + assert "anthropic/claude-opus-4-8" in _entry(options, "anthropic")["models"] async def test_a_bare_model_typed_for_codex_is_stored_so_it_finds_codex(fake_home: Path) -> None: @@ -406,7 +417,7 @@ def test_catalogue_ids_are_spelled_the_way_they_route() -> None: bare would be routed by keyword instead of to the provider the user picked. """ from raven.providers.common_models import litellm_models_for - from raven.providers.registry import find_by_name + from raven.providers.registry import find_by_model, find_by_name for slug in ("moonshot", "volcengine", "ollama_chat"): spec = find_by_name(slug) @@ -418,9 +429,14 @@ def test_catalogue_ids_are_spelled_the_way_they_route() -> None: # reads "moonshot/moonshot/x" as correct, so it could not tell a # re-prefixed id from a right one. head, _, rest = model.partition("/") - assert head == spec.model_prefix, f"{slug}: {model}" assert rest, f"{slug}: {model} has no id after the prefix" - assert not rest.startswith(f"{spec.model_prefix}/"), f"{slug}: double-prefixed {model}" + # The outcome, not one spelling of it: a candidate has to resolve + # back to the provider it was offered for. Asserting the wire prefix + # instead tied this to how the id happens to be spelled, which is + # `stored_model_id`'s business and differs from the routing prefix + # for every underscore-named provider. + assert find_by_model(model) is spec, f"{slug}: {model} resolves elsewhere" + assert not rest.startswith(f"{head}/"), f"{slug}: double-prefixed {model}" def test_catalogue_offers_only_chat_models() -> None: @@ -591,15 +607,22 @@ def test_the_account_catalogue_is_asked_only_when_it_can_answer( pytest.param("openai_codex", "gpt-5.6-sol", "openai-codex/gpt-5.6-sol", id="codex-typed-bare"), pytest.param("openai_codex", "openai-codex/gpt-5.4", "openai-codex/gpt-5.4", id="codex-typed-prefixed"), pytest.param("minimax_global", "MiniMax-M2", "minimax-global/MiniMax-M2", id="minimax-typed-bare"), - pytest.param("deepseek", "deepseek-chat", "deepseek-chat", id="a-provider-that-routes-on-it"), - pytest.param("azure_openai", "my-deployment", "my-deployment", id="azure-uses-it-verbatim-in-a-url"), + pytest.param("deepseek", "deepseek-chat", "deepseek/deepseek-chat", id="a-provider-that-routes-on-it"), + pytest.param("azure_openai", "my-deployment", "azure-openai/my-deployment", id="azure-names-its-provider-too"), + pytest.param("zai", "zhipu/glm-4.6", "zai/glm-4.6", id="a-former-name-is-canonicalized"), + pytest.param("zai", "openrouter/z-ai/glm-4.6", "openrouter/z-ai/glm-4.6", id="a-declared-skip-prefix-is-left"), ], ) def test_a_typed_model_is_stored_the_way_it_resolves_back(slug: str, typed: str, stored: str) -> None: """The add-model screen takes free text, and a bare id is claimed by keyword matching rather than by the provider it was entered under: "gpt-5.6-sol" - resolves to OpenAI. The listed models already carry the prefix; a typed one - has to end up spelled the same way.""" + resolves to OpenAI. + + Every provider now stores a qualified id, not the three whose own client + strips the prefix back off. Azure included: its deployment comes off again in + the URL builder, which is where that belongs -- storing it bare was the one + thing that made Azure ids shaped unlike everyone else's. + """ from raven.tui_rpc.methods.model import _stored_spelling assert _stored_spelling(slug, typed) == stored @@ -612,13 +635,35 @@ def test_every_provider_stores_a_model_id_that_finds_it_again(spec) -> None: whoever else claims the bare name. Providers that route on the prefix or use the id verbatim are covered by resolving as themselves. """ - from raven.providers.registry import find_by_model, needs_public_model_prefix + from raven.providers.registry import find_by_model from raven.tui_rpc.methods.model import _stored_spelling - if not needs_public_model_prefix(spec): - pytest.skip("no public prefix to add; the id is routed or used verbatim") - stored = _stored_spelling(spec.name, "some-model") resolved = find_by_model(stored) assert resolved is not None and resolved.name == spec.name, f"{stored} resolves to {resolved and resolved.name}" + + +async def test_a_user_written_overlay_reaches_the_picker(fake_home: Path) -> None: + """A model the catalogues cannot describe still arrives with a name. + + The list already let a model be added; naming one is what was missing, so a + self-hosted deployment reached the picker as a bare id with no description + line at all -- `_model_labels` skips every row nothing describes. + """ + _write_config( + fake_home, + { + "agents": {"defaults": {"model": "hosted-vllm/my-finetune-v3"}}, + "providers": { + "hosted_vllm": { + "apiBase": "http://localhost:8000/v1", + "models": ["hosted-vllm/my-finetune-v3"], + "modelOverlay": {"my-finetune-v3": {"label": "Our finetune", "description": "tuned on tickets"}}, + } + }, + }, + ) + entry = _entry(await model_options({}), "hosted_vllm") + label = (entry.get("model_labels") or {}).get("hosted-vllm/my-finetune-v3") + assert label == {"label": "Our finetune", "description": "tuned on tickets"} diff --git a/tests/test_tui_rpc_setup.py b/tests/test_tui_rpc_setup.py index dd7deaca..759bc33a 100644 --- a/tests/test_tui_rpc_setup.py +++ b/tests/test_tui_rpc_setup.py @@ -31,12 +31,35 @@ async def test_setup_status_provider_configured_true(fake_home: Path) -> None: cfg_dir = fake_home / ".raven" cfg_dir.mkdir() (cfg_dir / "config.json").write_text( - json.dumps({"agents": {"defaults": {"provider": "anthropic", "model": "anthropic/claude-sonnet-4-5"}}}) + json.dumps( + { + "agents": {"defaults": {"provider": "anthropic", "model": "anthropic/claude-sonnet-4-5"}}, + "providers": {"anthropic": {"apiKey": "sk-ant"}}, + } + ) ) result = await setup_status({}) assert result == {"provider_configured": True} +async def test_a_pinned_provider_name_is_not_credentials(fake_home: Path) -> None: + """The name says which section to ask about, not that it holds anything. + + ``agents.defaults.provider`` was waved through on its own, as a signal from + configs predating per-provider sections. It is now written on every model + change, so that branch would have let a pinned name stand for credentials + nobody has -- an empty config would pass the gate and the first turn would + fail with whatever the backend said about a missing key. + """ + cfg_dir = fake_home / ".raven" + cfg_dir.mkdir() + (cfg_dir / "config.json").write_text( + json.dumps({"agents": {"defaults": {"provider": "anthropic", "model": "anthropic/claude-sonnet-4-5"}}}) + ) + + assert await setup_status({}) == {"provider_configured": False} + + async def test_setup_status_provider_without_model_returns_false(fake_home: Path) -> None: # A provider but no default model can't drive a turn → not configured. cfg_dir = fake_home / ".raven" @@ -124,7 +147,12 @@ async def test_setup_status_registered_via_helper(fake_home: Path) -> None: cfg_dir = fake_home / ".raven" cfg_dir.mkdir() (cfg_dir / "config.json").write_text( - json.dumps({"agents": {"defaults": {"provider": "openai", "model": "openai/gpt-4o-mini"}}}) + json.dumps( + { + "agents": {"defaults": {"provider": "openai", "model": "openai/gpt-4o-mini"}}, + "providers": {"openai": {"apiKey": "sk-openai"}}, + } + ) ) d = Dispatcher() register_setup_methods(d) diff --git a/ui-tui/rpc-schema/openrpc.json b/ui-tui/rpc-schema/openrpc.json index 65a84d69..06df7a25 100644 --- a/ui-tui/rpc-schema/openrpc.json +++ b/ui-tui/rpc-schema/openrpc.json @@ -1413,7 +1413,22 @@ }, "total_models": { "type": "integer" }, "needs_api_base": { "type": "boolean" }, - "warning": { "type": "string" } + "warning": { "type": "string" }, + "model_labels": { + "type": "object", + "description": "Keyed by the model id as it appears in `models`.", + "additionalProperties": { "$ref": "#/components/schemas/ModelLabel" } + } + } + }, + "ModelLabel": { + "type": "object", + "additionalProperties": false, + "description": "How a model reads to a person. Absent for a model no catalogue knows -- one released since the bundled snapshot, or served by a local deployment -- in which case the id is all there is to show.", + "required": ["label"], + "properties": { + "label": { "type": "string" }, + "description": { "type": "string" } } }, "UsageSnapshot": { diff --git a/ui-tui/src/__tests__/gatewayTypesDrift.test.ts b/ui-tui/src/__tests__/gatewayTypesDrift.test.ts new file mode 100644 index 00000000..d5e7fcb6 --- /dev/null +++ b/ui-tui/src/__tests__/gatewayTypesDrift.test.ts @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: MIT +import { readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' + +/** + * The picker payload is declared twice: once generated from the OpenRPC schema + * (`rpc/generated.ts`), and once by hand in `gatewayTypes.ts`, which predates it + * and which the test stubs rely on for its all-optional fields. + * + * Two declarations of one contract drift, and the drift is silent in the + * direction that matters: a field added to the schema reaches the wire, the + * hand-written type does not know about it, and the component reading that type + * cannot see the data it is being sent. + * + * Merging them is a real change -- the generated shape has required fields the + * stubs do not build -- so until then this keeps the copy honest. + */ + +function propertyNames(source: string, interfaceName: string): Set { + const start = source.indexOf(`export interface ${interfaceName} {`) + if (start < 0) { + throw new Error(`${interfaceName} not found`) + } + const body = source.slice(start, source.indexOf('\n}', start)) + const names = new Set() + for (const line of body.split('\n').slice(1)) { + const match = /^\s{2}([A-Za-z_][A-Za-z0-9_]*)\??\s*:/.exec(line) + if (match) { + names.add(match[1]) + } + } + return names +} + +function read(relative: string): string { + return readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8') +} + +describe('ModelOptionProvider', () => { + it('declares every property the generated contract sends', () => { + const generated = propertyNames(read('../rpc/generated.ts'), 'ModelOptionProvider') + const handWritten = propertyNames(read('../gatewayTypes.ts'), 'ModelOptionProvider') + + expect(generated.size).toBeGreaterThan(0) + const missing = [...generated].filter(name => !handWritten.has(name)).sort() + expect(missing, `add these to gatewayTypes.ts ModelOptionProvider: ${missing.join(', ')}`).toEqual([]) + }) +}) diff --git a/ui-tui/src/__tests__/modelPicker.test.tsx b/ui-tui/src/__tests__/modelPicker.test.tsx index 54751b0c..ab119c9b 100644 --- a/ui-tui/src/__tests__/modelPicker.test.tsx +++ b/ui-tui/src/__tests__/modelPicker.test.tsx @@ -653,3 +653,44 @@ describe('ModelPicker', () => { h.unmount() }) }) + +describe('model labels', () => { + const described: ModelOptionProvider = { + ...anthropic, + model_labels: { + 'claude-sonnet-4-6': { + context: 1000000, + description: 'Claude workhorse for coding agents', + label: 'Claude Sonnet 4.6' + } + }, + models: ['claude-sonnet-4-6', 'some-unlisted-finetune'], + total_models: 2 + } + + it('shows a name and a description beside the id, and the id alone without one', async () => { + const h = mount([described]) + await waitForFrame(h, 'Anthropic') + await h.type(ENTER) + await waitForFrame(h, 'step 2/2') + + const frame = normalize(h.frame()) + // The id stays: it is what gets stored, and it is what a vendor's docs name. + expect(frame).toContain('Claude Sonnet 4.6 · claude-sonnet-4-6') + expect(frame).toContain('Claude workhorse for coding agents') + // No catalogue knows a local finetune, and the row still has to render. + expect(frame).toContain('some-unlisted-finetune') + + h.unmount() + }) + + it('renders ids unchanged for a provider the payload describes nothing for', async () => { + const h = mount([anthropic]) + await waitForFrame(h, 'Anthropic') + await h.type(ENTER) + await waitForFrame(h, 'step 2/2') + + expect(normalize(h.frame())).toContain('claude-sonnet-4-6') + h.unmount() + }) +}) diff --git a/ui-tui/src/components/modelPicker.tsx b/ui-tui/src/components/modelPicker.tsx index 6610d7f9..adeb9e4e 100644 --- a/ui-tui/src/components/modelPicker.tsx +++ b/ui-tui/src/components/modelPicker.tsx @@ -1012,6 +1012,11 @@ export function ModelPicker({ gw, launcher, onCancel, onSelect, sessionId, suspe } const prefix = modelIdx === idx ? '▸ ' : row === currentModel ? '* ' : ' ' + // The id stays: it is what gets stored, and a user comparing it against + // a vendor's docs needs to see it. The name goes first because that is + // what someone choosing a model is reading for. + const label = provider?.model_labels?.[row]?.label + const text = label && label !== row ? `${label} · ${row}` : row return ( {prefix} - {idx + 1}. {row} + {idx + 1}. {text} ) })} @@ -1031,6 +1036,13 @@ export function ModelPicker({ gw, launcher, onCancel, onSelect, sessionId, suspe {offset + VISIBLE < models.length ? ` ↓ ${models.length - offset - VISIBLE} more` : ' '} + {/* One line about the highlighted model. Blank rather than absent, so the + list below does not jump as the cursor moves between a model the + catalogue describes and one it does not. */} + + {provider?.model_labels?.[models[modelIdx] ?? '']?.description ?? ' '} + + scope: global diff --git a/ui-tui/src/gatewayTypes.ts b/ui-tui/src/gatewayTypes.ts index 612d003b..9e1d5648 100644 --- a/ui-tui/src/gatewayTypes.ts +++ b/ui-tui/src/gatewayTypes.ts @@ -336,11 +336,18 @@ export interface ToolsConfigureResponse { // ── Model picker ───────────────────────────────────────────────────── +// A hand-written copy of the picker payload that predates the generated RPC +// types. Kept because the test stubs build partial objects that the generated +// shape, whose fields are required, rejects. Two declarations of one contract +// drift, and the drift is silent: a field added to the schema but not here +// arrives on the wire invisible to the component reading this type. The drift +// test beside this file fails if a generated property is missing here. export interface ModelOptionProvider { auth_type?: string authenticated?: boolean is_current?: boolean key_env?: null | string + model_labels?: Record models?: string[] name: string needs_api_base?: boolean diff --git a/ui-tui/src/rpc/generated.ts b/ui-tui/src/rpc/generated.ts index d8fa6050..9abe587c 100644 --- a/ui-tui/src/rpc/generated.ts +++ b/ui-tui/src/rpc/generated.ts @@ -152,6 +152,22 @@ export interface ModelOptionProvider { total_models: number; needs_api_base: boolean; warning: string; + /** + * Keyed by the model id as it appears in `models`. + */ + model_labels?: { + [k: string]: ModelLabel; + }; +} +/** + * How a model reads to a person. Absent for a model no catalogue knows -- one released since the bundled snapshot, or served by a local deployment -- in which case the id is all there is to show. + * + * This interface was referenced by `RavenRpcRoot`'s JSON-Schema + * via the `definition` "ModelLabel". + */ +export interface ModelLabel { + label: string; + description?: string; } /** * This interface was referenced by `RavenRpcRoot`'s JSON-Schema From 95e7e37839c01ad4ee2a84b9f678c38797ad9483 Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 20:02:04 +0800 Subject: [PATCH 02/78] fix(providers): keep codex sse error details for retry classification The error and response.failed SSE events were collapsed into a bare "Codex response failed" RuntimeError, so classify_error landed in the unknown bucket (retryable=False) and an overloaded backend was never retried. Carry the structured code and message into the exception text; the existing "overloaded" substring needle then classifies server_is_overloaded as a retryable server error without touching the classifier. Co-authored-by: Claude (claude-fable-5) --- raven/providers/openai_codex_provider.py | 13 ++++- tests/test_openai_codex_provider.py | 62 ++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/raven/providers/openai_codex_provider.py b/raven/providers/openai_codex_provider.py index 39f58edd..cca3e99c 100644 --- a/raven/providers/openai_codex_provider.py +++ b/raven/providers/openai_codex_provider.py @@ -394,7 +394,18 @@ async def _consume_sse(response: httpx.Response, timeout: float) -> tuple[str, l status = (event.get("response") or {}).get("status") finish_reason = _map_finish_reason(status) elif event_type in {"error", "response.failed"}: - raise RuntimeError("Codex response failed") + # The code is the retry signal: classify_error buckets by message + # substring, and "server_is_overloaded" is what turns a dead-end + # unknown into a retryable server error. An `error` event carries + # it at the top level or under "error"; `response.failed` nests it + # under the response. + err = event.get("error") or (event.get("response") or {}).get("error") or {} + if not isinstance(err, dict): + err = {} + code = err.get("code") or event.get("code") or "" + message = err.get("message") or event.get("message") or "" + detail = ": ".join(str(part) for part in (code, message) if part) + raise RuntimeError(f"Codex response failed: {detail}" if detail else "Codex response failed") return content, tool_calls, finish_reason diff --git a/tests/test_openai_codex_provider.py b/tests/test_openai_codex_provider.py index 917711a8..779b3edc 100644 --- a/tests/test_openai_codex_provider.py +++ b/tests/test_openai_codex_provider.py @@ -8,6 +8,7 @@ from __future__ import annotations import asyncio +import json import pytest @@ -15,6 +16,7 @@ DEFAULT_CODEX_URL, OpenAICodexProvider, _build_headers, + _consume_sse, _convert_messages, _convert_tool_output, _iter_sse, @@ -70,6 +72,66 @@ async def test_iter_sse_per_event_idle_timeout_raises(): assert events == [{"type": "ping"}] +@pytest.mark.asyncio +async def test_consume_sse_error_event_keeps_the_structured_code_and_message(): + """The code is the retry signal: without it, an overloaded backend looks + like an unclassifiable error instead of a retryable one.""" + event = { + "type": "error", + "code": "server_is_overloaded", + "message": "Our servers are currently overloaded. Please try again later.", + } + resp = _FakeStreamResponse([f"data: {json.dumps(event)}", ""]) + + with pytest.raises(RuntimeError) as exc_info: + await _consume_sse(resp, timeout=1.0) + + assert "server_is_overloaded" in str(exc_info.value) + assert "Our servers are currently overloaded. Please try again later." in str(exc_info.value) + + +@pytest.mark.asyncio +async def test_consume_sse_response_failed_event_keeps_the_nested_error(): + """response.failed nests the same error shape under "response" instead of + at the event's top level.""" + event = { + "type": "response.failed", + "response": { + "status": "failed", + "error": { + "code": "server_is_overloaded", + "message": "Our servers are currently overloaded.", + }, + }, + } + resp = _FakeStreamResponse([f"data: {json.dumps(event)}", ""]) + + with pytest.raises(RuntimeError) as exc_info: + await _consume_sse(resp, timeout=1.0) + + assert "server_is_overloaded" in str(exc_info.value) + assert "Our servers are currently overloaded." in str(exc_info.value) + + +def test_consume_sse_error_classifies_as_retryable_server_error(): + """Closes the loop: the RuntimeError raised for a codex error event must + still land classify_error in the retryable "server" bucket, not unknown.""" + event = { + "type": "error", + "code": "server_is_overloaded", + "message": "Our servers are currently overloaded.", + } + resp = _FakeStreamResponse([f"data: {json.dumps(event)}", ""]) + + with pytest.raises(RuntimeError) as exc_info: + asyncio.run(_consume_sse(resp, timeout=1.0)) + classification = OpenAICodexProvider.classify_error(exc_info.value) + + assert classification.category == "server" + assert classification.retryable is True + assert classification.should_fallback is True + + _TINY_PNG_URI = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==" From 46e8538eac71cc80f96aed187992d8c0713210a8 Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 20:31:01 +0800 Subject: [PATCH 03/78] fix(*): wire the agent model into skill forge rewriter and gate The rewriter had no model parameter and the gate only its dedicated llm_gate_model, so both fell through to provider.default_model on every call -- the only auxiliary LLM calls in the repo not chained back to the configured agent model. Thread build_context_engine's model into _build_rewriter_and_gate: the rewriter now sends it explicitly and the gate falls back to it when llm_gate_model is unset. Co-authored-by: Claude (claude-fable-5) --- raven/context_engine/factory.py | 5 ++- raven/memory_engine/skill_forge/rewriter.py | 3 ++ tests/test_context_engine_factory.py | 45 ++++++++++++++++++++- tests/test_skill_forge_gate.py | 7 ++++ tests/test_skill_forge_rewriter.py | 12 ++++++ 5 files changed, 70 insertions(+), 2 deletions(-) diff --git a/raven/context_engine/factory.py b/raven/context_engine/factory.py index 11f522d3..7151324f 100644 --- a/raven/context_engine/factory.py +++ b/raven/context_engine/factory.py @@ -108,6 +108,7 @@ def build_context_engine( rewriter, gate = _build_rewriter_and_gate( provider=provider, + model=model, skill_forge_config=skill_forge_config, skill_forge_router_config=skill_forge_router_config, ) @@ -216,6 +217,7 @@ def _build_router( def _build_rewriter_and_gate( *, provider: LLMProvider, + model: str, skill_forge_config: "SkillForgeConfig | None", skill_forge_router_config: "SkillForgeRouterConfig", ) -> "tuple[QueryRewriter | None, LLMGateFilter | None]": @@ -236,6 +238,7 @@ def _build_rewriter_and_gate( if bool(getattr(skill_forge_config, "rewrite_enabled", False)): rewriter = QueryRewriter( provider, + model=model, max_tokens=int(getattr(skill_forge_config, "rewrite_max_tokens", 8192) or 8192), ) @@ -248,7 +251,7 @@ def _build_rewriter_and_gate( provider, max_select=int(getattr(skill_forge_config, "llm_gate_max_select", 2) or 2), legacy_top_k=int(skill_forge_router_config.top_k or 5), - model=getattr(skill_forge_config, "llm_gate_model", None) or None, + model=getattr(skill_forge_config, "llm_gate_model", None) or model or None, temperature=float(getattr(skill_forge_config, "llm_gate_temperature", 0.0)), max_tokens=int(getattr(skill_forge_config, "llm_gate_max_tokens", 8192) or 8192), ) diff --git a/raven/memory_engine/skill_forge/rewriter.py b/raven/memory_engine/skill_forge/rewriter.py index 5c3dd3b6..e0018aec 100644 --- a/raven/memory_engine/skill_forge/rewriter.py +++ b/raven/memory_engine/skill_forge/rewriter.py @@ -67,10 +67,12 @@ def __init__( self, provider: "LLMProvider", *, + model: str | None = None, max_tokens: int = 8192, temperature: float = 0.3, ) -> None: self._provider = provider + self._model = model self._max_tokens = max_tokens self._temperature = temperature @@ -96,6 +98,7 @@ async def analyze(self, query: str) -> RewriteResult: resp = await asyncio.wait_for( self._provider.chat_with_retry( messages=[{"role": "user", "content": prompt}], + model=self._model or None, max_tokens=self._max_tokens, temperature=self._temperature, ), diff --git a/tests/test_context_engine_factory.py b/tests/test_context_engine_factory.py index 505c4a31..1d46a5ed 100644 --- a/tests/test_context_engine_factory.py +++ b/tests/test_context_engine_factory.py @@ -25,6 +25,7 @@ ContextConfig, HubSourceConfig, MemoryConfig, + SkillForgeConfig, SkillForgeRouterConfig, ) from raven.context_engine import ContextAssembler @@ -81,6 +82,8 @@ def _build_engine( backend=None, hub_endpoint: str | None = None, memory_config: MemoryConfig | None = None, + model: str = "stub", + skill_forge_config: SkillForgeConfig | None = None, ) -> ContextAssembler: builder = ContextBuilder(workspace=tmp_path) engine = build_context_engine( @@ -88,7 +91,7 @@ def _build_engine( config=ContextConfig(), builder=builder, provider=_StubProvider(), - model="stub", + model=model, context_window_tokens=8192, get_tool_definitions=_stub_get_defs, backend=backend, @@ -96,6 +99,7 @@ def _build_engine( skill_forge_router_config=SkillForgeRouterConfig( hub=HubSourceConfig(endpoint=hub_endpoint), ), + skill_forge_config=skill_forge_config, ) assert isinstance(engine, ContextAssembler) return engine @@ -166,6 +170,45 @@ def test_track_ids_from_memory_config(self, tmp_path: Path) -> None: assert everos._agent_id == "robo" +# --------------------------------------------------------------------------- +# Rewriter / gate model wiring — both must follow the agent's main model +# unless the gate has its own dedicated override. +# --------------------------------------------------------------------------- + + +class TestRewriterGateModelWiring: + def test_rewriter_receives_build_context_engine_model(self, tmp_path: Path) -> None: + engine = _build_engine( + tmp_path, + model="main-model", + skill_forge_config=SkillForgeConfig(rewrite_enabled=True, llm_gate_enabled=False), + ) + skills = next(b for b in engine._builders if isinstance(b, SkillsSegmentBuilder)) + assert skills._rewriter._model == "main-model" + + def test_gate_falls_back_to_main_model_when_llm_gate_model_unset(self, tmp_path: Path) -> None: + engine = _build_engine( + tmp_path, + model="main-model", + skill_forge_config=SkillForgeConfig(rewrite_enabled=False, llm_gate_enabled=True, llm_gate_model=None), + ) + skills = next(b for b in engine._builders if isinstance(b, SkillsSegmentBuilder)) + assert skills._gate._model == "main-model" + + def test_gate_prefers_dedicated_llm_gate_model(self, tmp_path: Path) -> None: + engine = _build_engine( + tmp_path, + model="main-model", + skill_forge_config=SkillForgeConfig( + rewrite_enabled=False, + llm_gate_enabled=True, + llm_gate_model="gate-only-model", + ), + ) + skills = next(b for b in engine._builders if isinstance(b, SkillsSegmentBuilder)) + assert skills._gate._model == "gate-only-model" + + # --------------------------------------------------------------------------- # AgentLoop helpers # --------------------------------------------------------------------------- diff --git a/tests/test_skill_forge_gate.py b/tests/test_skill_forge_gate.py index fa029397..a09e6f16 100644 --- a/tests/test_skill_forge_gate.py +++ b/tests/test_skill_forge_gate.py @@ -143,3 +143,10 @@ async def test_tools_block_absent_when_tools_none() -> None: await LLMGateFilter(provider).filter("task", [_hit("local/a", "a")]) prompt = provider.calls[0]["messages"][0]["content"] assert "# Agent Tools" not in prompt + + +async def test_filter_passes_explicit_model_to_provider() -> None: + provider = _StubProvider(json.dumps({"plan": "p", "skills": []})) + gate = LLMGateFilter(provider, model="gpt-4o") + await gate.filter("task", [_hit("local/a", "a")]) + assert provider.calls[0]["model"] == "gpt-4o" diff --git a/tests/test_skill_forge_rewriter.py b/tests/test_skill_forge_rewriter.py index ffa6b265..760409fe 100644 --- a/tests/test_skill_forge_rewriter.py +++ b/tests/test_skill_forge_rewriter.py @@ -97,3 +97,15 @@ async def test_analyze_finish_reason_error_defaults_to_retrieval() -> None: provider = _StubProvider(_Resp(content="", finish_reason="error")) result = await QueryRewriter(provider).analyze("q") assert result.need_retrieval is True + + +async def test_analyze_passes_model_to_provider() -> None: + provider = _StubProvider(json.dumps({"need_retrieval": False})) + await QueryRewriter(provider, model="gpt-4o").analyze("hello there") + assert provider.calls[0]["model"] == "gpt-4o" + + +async def test_analyze_no_model_passes_none() -> None: + provider = _StubProvider(json.dumps({"need_retrieval": False})) + await QueryRewriter(provider).analyze("hello there") + assert provider.calls[0]["model"] is None From f885247d4ab3f663bf6f052763714758adfc5907 Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 20:54:52 +0800 Subject: [PATCH 04/78] fix(*): resolve the context window through one ladder in providers The configured default 65536 flowed raw into trimming, token budgets, memory consolidation, and the TUI gauge, so a model with a 200k window lost two thirds of it every turn and the gauge rendered against a number that was nobody's. The ladder now lives in providers/rates.py effective_context_window: explicit config wins, then the model's real window, then a documented fallback. AgentLoop resolves once at construction and pins explicit values (a pin now also survives the per-call live lookup), /model switches re-walk the ladder, and an unknown window reports context_max=0 so the UI shows its empty state instead of a guess. Known gap, unchanged scope: trimmer and consolidator keep their construction-time snapshot after a model switch. Co-authored-by: Claude (claude-fable-5) --- benchmarks/clawbench/stream.py | 5 +- raven/agent/loop/main.py | 37 +++++++-- raven/cli/doctor_commands.py | 4 +- raven/config/schema.py | 5 +- raven/providers/rates.py | 23 ++++++ raven/tui_rpc/methods/config.py | 1 + raven/tui_rpc/methods/session.py | 21 ++--- tests/test_agent_loop_usage_sink.py | 98 +++++++++++++++++++++-- tests/test_config_schema.py | 25 ++++++ tests/test_provider_rates.py | 23 ++++++ tests/test_tui_rpc_config.py | 3 + tests/test_tui_rpc_session_init_bundle.py | 42 ++++++---- 12 files changed, 244 insertions(+), 43 deletions(-) create mode 100644 tests/test_config_schema.py diff --git a/benchmarks/clawbench/stream.py b/benchmarks/clawbench/stream.py index 0eed02cb..7f88d125 100644 --- a/benchmarks/clawbench/stream.py +++ b/benchmarks/clawbench/stream.py @@ -101,6 +101,7 @@ def __init__( from raven.cli.commands import _make_provider from raven.config.loader import load_config, set_config_path from raven.config.raven import ContextConfig + from raven.providers.rates import effective_context_window from raven.session.manager import SessionManager workspace.mkdir(parents=True, exist_ok=True) @@ -126,7 +127,9 @@ def __init__( self.config.agents.defaults.workspace = str(workspace.resolve()) self.provider = UsageTrackingProvider(_make_provider(self.config)) self.model = self.config.agents.defaults.model - self.context_window = int(context_window or self.config.agents.defaults.context_window_tokens) + self.context_window = effective_context_window( + self.model, context_window or self.config.agents.defaults.context_window_tokens + ) self.curator_model = curator_model or self.model self.session_id = session_id self.previous_totals = dict(self.provider.accumulated) diff --git a/raven/agent/loop/main.py b/raven/agent/loop/main.py index afe8247d..6cabf2c8 100644 --- a/raven/agent/loop/main.py +++ b/raven/agent/loop/main.py @@ -45,7 +45,7 @@ from raven.memory_engine.consolidate.consolidator import MemoryConsolidator, MemoryStore from raven.providers.base import LLMProvider, LLMResponse, ToolCallRequest from raven.providers.capabilities import image_placeholder_text, supports_image_tool_result, vision_verdict -from raven.providers.rates import resolve_context_window +from raven.providers.rates import effective_context_window, resolve_context_window from raven.sandbox import SandboxConfig, SandboxExecutor, SandboxInitError, build_executor from raven.session.manager import Session, SessionManager from raven.spine.turn import Origin @@ -290,7 +290,7 @@ def __init__( workspace: Path, model: str | None = None, max_iterations: int = 40, - context_window_tokens: int = 65_536, + context_window_tokens: int | None = None, brave_api_key: str | None = None, web_proxy: str | None = None, exec_config: ExecToolConfig | None = None, @@ -377,7 +377,10 @@ def __init__( self.max_iterations = max_iterations # Empty-response recovery budgets. None → enabled defaults. self._recovery_limits = empty_recovery if empty_recovery is not None else RecoveryLimits() - self.context_window_tokens = context_window_tokens + # A caller that passed a positive value pinned the window; None/0 means + # "figure it out", resolved once here against the model's real window. + self._context_window_pinned = bool(context_window_tokens) + self.context_window_tokens = context_window_tokens or effective_context_window(self.model, None) self.brave_api_key = brave_api_key self.jina_api_key = jina_api_key self.web_proxy = web_proxy @@ -464,7 +467,7 @@ def __init__( builder=self.context, provider=provider, model=self.model, - context_window_tokens=context_window_tokens, + context_window_tokens=self.context_window_tokens, get_tool_definitions=self.tools.get_definitions, now_fn=now_fn, # The factory uses these to assemble the unified engine's @@ -546,7 +549,7 @@ def __init__( provider=provider, model=self.model, sessions=self.sessions, - context_window_tokens=context_window_tokens, + context_window_tokens=self.context_window_tokens, build_messages=self.context.build_messages, get_tool_definitions=self.tools.get_definitions, now_fn=now_fn, @@ -693,6 +696,18 @@ def configure_personalization(self, enable: bool) -> None: self.enable_personalization = enable logger.info("Personalization flow: {}", "enabled" if enable else "disabled") + def refresh_context_window(self) -> None: + """Re-resolve ``context_window_tokens`` against the current ``self.model``. + + A no-op once the window was pinned at construction -- a pin is a + deliberate override, and a model switch afterwards must not quietly + discard it. Unpinned loops re-walk the ladder so a ``/model`` switch + picks up the new model's real window instead of keeping the old one's. + """ + if self._context_window_pinned: + return + self.context_window_tokens = effective_context_window(self.model, None) + def _register_default_tools(self) -> None: """Register the default set of tools.""" allowed_dir = self.workspace if self.restrict_to_workspace else None @@ -1886,9 +1901,15 @@ async def _run_agent_loop( if usage_sink is not None and response.usage: prompt_tokens = int(response.usage.get("prompt_tokens", 0) or 0) completion_tokens = int(response.usage.get("completion_tokens", 0) or 0) - # Real window from the model's provider table when LiteLLM lags - # (e.g. OpenRouter); otherwise the configured default. - context_max = resolve_context_window(call_model) or self.context_window_tokens + # A pin always wins over the live table -- that is what pinning + # means. Unpinned, the live window from the model's provider + # table (e.g. OpenRouter, when LiteLLM lags) answers instead; + # unknown to that table too, 0 tells the UI to show its empty + # state rather than a number that isn't this model's. + if self._context_window_pinned: + context_max = self.context_window_tokens + else: + context_max = resolve_context_window(call_model) or 0 context_used = prompt_tokens + completion_tokens usage_sink.clear() usage_sink["prompt_tokens"] = prompt_tokens diff --git a/raven/cli/doctor_commands.py b/raven/cli/doctor_commands.py index 57d9c887..7a637ae9 100644 --- a/raven/cli/doctor_commands.py +++ b/raven/cli/doctor_commands.py @@ -42,7 +42,7 @@ class RoutingInfo: model: str provider: Optional[str] max_tokens: int - context_window_tokens: int + context_window_tokens: Optional[int] @dataclass @@ -328,7 +328,7 @@ def _render_human_output(report: DoctorReport) -> None: else: console.print(" Routes to: [red][/red]") console.print(f" Max tokens: {routing.max_tokens}") - console.print(f" Context win: {routing.context_window_tokens}") + console.print(f" Context win: {routing.context_window_tokens if routing.context_window_tokens else 'auto'}") features = report.features if features is not None: diff --git a/raven/config/schema.py b/raven/config/schema.py index 989f874e..2e5dca38 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -249,7 +249,10 @@ class AgentDefaults(Base): model: str = "anthropic/claude-opus-4-5" provider: str = "auto" # Provider name (e.g. "anthropic", "openrouter") or "auto" for auto-detection max_tokens: int = 8192 - context_window_tokens: int = 65_536 + # None (or 0) means "figure it out" -- resolved against the model's real + # window at construction time. A positive value pins the window, taking + # priority over whatever the model's own catalogue reports. + context_window_tokens: int | None = None temperature: float = 0.1 # Per-call wall-clock cap (seconds) for every LLM request (main loop and # sub-agents). Bounds a stalled backend that trickles bytes without ever diff --git a/raven/providers/rates.py b/raven/providers/rates.py index 50a2d199..0f5abb3b 100644 --- a/raven/providers/rates.py +++ b/raven/providers/rates.py @@ -29,6 +29,10 @@ from raven.providers import model_catalog_cache +#: One home for the window ladder's documented fallback: an unknown model gets +#: this many tokens of headroom rather than a number invented at the call site. +DEFAULT_CONTEXT_WINDOW_TOKENS = 65_536 + #: Rate pair: (prompt_cost_per_token, completion_cost_per_token) in USD. #: Keep this table small -- it is a fallback for brand-new models that LiteLLM #: has not indexed yet. Check LiteLLM first before adding here. @@ -494,6 +498,25 @@ def resolve_context_window(model: str) -> int | None: return None +def effective_context_window(model: str, configured: int | None) -> int: + """The context window to size trimming with -- the decision ladder's front door. + + Explicit configuration wins outright: a user or caller who pinned a number + meant it as an override, not a hint. Absent that, the model's real window + from ``resolve_context_window`` answers; absent *that* too (an unmapped + model, or every source down), this module's documented default does -- + never ``None``, since the caller is about to size a request with the + result. + + ``resolve_context_window`` already folds every LiteLLM and network failure + into ``None`` rather than raising (see its tiers), so there is no + exception left here to catch. + """ + if configured: + return configured + return resolve_context_window(model) or DEFAULT_CONTEXT_WINDOW_TOKENS + + def reset_openrouter_cache() -> None: """Clear the in-process OpenRouter catalog cache. diff --git a/raven/tui_rpc/methods/config.py b/raven/tui_rpc/methods/config.py index 37fc9875..94adc43f 100644 --- a/raven/tui_rpc/methods/config.py +++ b/raven/tui_rpc/methods/config.py @@ -360,6 +360,7 @@ def _set_model( # subagent manager, the context engine and the consolidator at build # time, and each keeps it. set_provider is what reaches them. loop.set_provider(built_provider, raw_value) + loop.refresh_context_window() return {"applied": True, "previous": previous, "value": raw_value} diff --git a/raven/tui_rpc/methods/session.py b/raven/tui_rpc/methods/session.py index d86a474a..459f4a0c 100644 --- a/raven/tui_rpc/methods/session.py +++ b/raven/tui_rpc/methods/session.py @@ -104,8 +104,10 @@ def _baseline_usage( All counters are zero at session.create: a fresh session_key carries no prior LLM calls. Each turn's ``message.complete`` event updates them - post-turn. ``context_max`` is the model's real window — live from the - provider table when LiteLLM lags (e.g. OpenRouter), else config default. + post-turn. ``context_max`` follows the same ladder ``AgentLoop`` uses: a + pinned ``context_window_tokens`` wins outright; otherwise the model's real + window (live from the provider table when LiteLLM lags, e.g. OpenRouter), + or 0 when neither is known — the UI's empty state, not a borrowed number. Usage starts at zero for a fresh session by design. Resume reuses the zero baseline; counters refresh on the next turn. @@ -115,12 +117,12 @@ def _baseline_usage( """ from raven.providers.rates import is_plan_billed - context_max = config.agents.defaults.context_window_tokens model = getattr(agent_loop, "model", None) - if model: - live_window = resolve_context_window(model) - if live_window: - context_max = live_window + configured = config.agents.defaults.context_window_tokens + if configured: + context_max = configured + else: + context_max = (resolve_context_window(model) if model else None) or 0 return { "input": 0, "output": 0, @@ -142,15 +144,16 @@ def _default_session_info( zero usage, ``lazy=True``); version is always real (cached at module load). """ model_id = config.agents.defaults.model + usage = _baseline_usage(agent_loop, config) info: dict[str, Any] = { "model": model_id, "model_id": model_id, "provider": config.agents.defaults.provider, - "context_window": config.agents.defaults.context_window_tokens, + "context_window": usage["context_max"], "lazy": agent_loop is None, "skills": _enumerate_skills(agent_loop), "tools": _enumerate_tools(agent_loop), - "usage": _baseline_usage(agent_loop, config), + "usage": usage, "version": _RAVEN_VERSION, "cwd": os.getcwd(), "mcp_servers": [], diff --git a/tests/test_agent_loop_usage_sink.py b/tests/test_agent_loop_usage_sink.py index 66d91359..e9e716f8 100644 --- a/tests/test_agent_loop_usage_sink.py +++ b/tests/test_agent_loop_usage_sink.py @@ -66,14 +66,17 @@ def _reset_openrouter_cache(): rates._OPENROUTER_CACHE.clear() -def _make_agent(workspace: Path, provider: LLMProvider, model: str, window: int) -> AgentLoop: +def _make_agent(workspace: Path, provider: LLMProvider, model: str, window: int | None) -> AgentLoop: + kwargs: dict = {} + if window is not None: + kwargs["context_window_tokens"] = window return AgentLoop( provider=provider, workspace=workspace, model=model, max_iterations=2, - context_window_tokens=window, restrict_to_workspace=True, + **kwargs, ) @@ -100,13 +103,12 @@ async def test_usage_sink_carries_context_gauge_and_cost(workspace): assert "cost_usd" in sink -@pytest.mark.asyncio -async def test_usage_sink_context_max_from_live_openrouter(workspace, monkeypatch): - """An OpenRouter model LiteLLM lags on gets its real window from /models.""" +def _patch_live_openrouter_window(monkeypatch, window: int) -> None: + """Route the OpenRouter models fetch to report ``window`` for deepseek-v4-pro.""" models = [ { "id": "deepseek/deepseek-v4-pro", - "context_length": 163840, + "context_length": window, "pricing": {"prompt": "0.0000005", "completion": "0.0000015"}, } ] @@ -124,12 +126,18 @@ def client_factory(*args, **kwargs): monkeypatch.setattr(rates.httpx, "Client", client_factory) monkeypatch.setattr(rates, "_OPENROUTER_CACHE_TIME", 0.0) + +@pytest.mark.asyncio +async def test_usage_sink_context_max_from_live_openrouter(workspace, monkeypatch): + """Unpinned, an OpenRouter model LiteLLM lags on gets its real window from /models.""" + _patch_live_openrouter_window(monkeypatch, 163840) + provider = UsageProvider("openrouter/deepseek/deepseek-v4-pro", 1000, 500) agent = _make_agent( workspace, provider, model="openrouter/deepseek/deepseek-v4-pro", - window=8192, + window=None, ) sink: dict = {} @@ -145,3 +153,79 @@ def client_factory(*args, **kwargs): assert sink["context_max"] == 163840 assert sink["context_used"] == 1500 + + +@pytest.mark.asyncio +async def test_usage_sink_context_max_stays_pinned_over_live_openrouter(workspace, monkeypatch): + """A pinned window wins even when the model's live window disagrees.""" + _patch_live_openrouter_window(monkeypatch, 163840) + + provider = UsageProvider("openrouter/deepseek/deepseek-v4-pro", 1000, 500) + agent = _make_agent( + workspace, + provider, + model="openrouter/deepseek/deepseek-v4-pro", + window=8192, + ) + sink: dict = {} + + await agent._process_message( + TurnRequest( + origin=Origin.USER, + source=Source(channel="test", chat_id="c1", sender_id="user", chat_type=ChatType.DM), + text="hi", + ), + session_key="s1", + usage_sink=sink, + ) + + assert sink["context_max"] == 8192 + assert sink["context_used"] == 1500 + + +# --------------------------------------------------------------------------- # +# construction-time ladder: _context_window_pinned + refresh_context_window # +# --------------------------------------------------------------------------- # + + +def test_no_configured_window_resolves_via_the_ladder_and_is_unpinned(workspace): + """An unresolvable model falls back to the documented default, unpinned.""" + provider = UsageProvider("stub", 0, 0) + agent = _make_agent(workspace, provider, model="stub", window=None) + + assert agent._context_window_pinned is False + assert agent.context_window_tokens == rates.DEFAULT_CONTEXT_WINDOW_TOKENS + + +def test_a_configured_window_is_pinned_at_construction(workspace): + provider = UsageProvider("stub", 0, 0) + agent = _make_agent(workspace, provider, model="stub", window=8192) + + assert agent._context_window_pinned is True + assert agent.context_window_tokens == 8192 + + +def test_refresh_context_window_is_a_noop_once_pinned(workspace, monkeypatch): + """A pin is a deliberate override; a later model switch must not discard it.""" + _patch_live_openrouter_window(monkeypatch, 163840) + + provider = UsageProvider("stub", 0, 0) + agent = _make_agent(workspace, provider, model="stub", window=8192) + + agent.model = "openrouter/deepseek/deepseek-v4-pro" + agent.refresh_context_window() + + assert agent.context_window_tokens == 8192 + + +def test_refresh_context_window_follows_the_new_model_when_unpinned(workspace, monkeypatch): + """Unpinned, a ``/model`` switch re-walks the ladder for the new model.""" + provider = UsageProvider("stub", 0, 0) + agent = _make_agent(workspace, provider, model="stub", window=None) + assert agent.context_window_tokens == rates.DEFAULT_CONTEXT_WINDOW_TOKENS + + _patch_live_openrouter_window(monkeypatch, 163840) + agent.model = "openrouter/deepseek/deepseek-v4-pro" + agent.refresh_context_window() + + assert agent.context_window_tokens == 163840 diff --git a/tests/test_config_schema.py b/tests/test_config_schema.py new file mode 100644 index 00000000..458bfe09 --- /dev/null +++ b/tests/test_config_schema.py @@ -0,0 +1,25 @@ +"""Tests for ``raven.config.schema.AgentDefaults.context_window_tokens``. + +None (or 0) means "figure it out" against the model's real window; a positive +value pins it. See ``raven.providers.rates.effective_context_window`` for the +ladder that reads this field. +""" + +from __future__ import annotations + +from raven.config.schema import AgentDefaults + + +def test_context_window_tokens_defaults_to_none() -> None: + assert AgentDefaults().context_window_tokens is None + + +def test_context_window_tokens_explicit_value_round_trips() -> None: + defaults = AgentDefaults(context_window_tokens=200_000) + assert defaults.context_window_tokens == 200_000 + + +def test_context_window_tokens_camel_alias_round_trips() -> None: + defaults = AgentDefaults.model_validate({"contextWindowTokens": 200_000}) + assert defaults.context_window_tokens == 200_000 + assert defaults.model_dump(by_alias=True)["contextWindowTokens"] == 200_000 diff --git a/tests/test_provider_rates.py b/tests/test_provider_rates.py index 099c8410..be32b14d 100644 --- a/tests/test_provider_rates.py +++ b/tests/test_provider_rates.py @@ -301,6 +301,29 @@ def test_a_window_unknown_to_every_source_is_none(monkeypatch): assert resolve_context_window("openrouter/some/model-not-listed") is None +# --- effective_context_window: explicit config > real window > fallback --- + + +def test_a_configured_window_wins_even_when_the_real_one_disagrees(monkeypatch): + """A pin is an override, so it answers even when the model has a real window.""" + monkeypatch.setattr(rates, "resolve_context_window", lambda model: 200_000) + + assert rates.effective_context_window("anthropic/claude-sonnet-4-5", 40_000) == 40_000 + + +def test_no_configured_window_falls_back_to_the_real_one(monkeypatch): + monkeypatch.setattr(rates, "resolve_context_window", lambda model: 200_000) + + assert rates.effective_context_window("anthropic/claude-sonnet-4-5", None) == 200_000 + assert rates.effective_context_window("anthropic/claude-sonnet-4-5", 0) == 200_000 + + +def test_neither_configured_nor_resolvable_uses_the_documented_default(monkeypatch): + monkeypatch.setattr(rates, "resolve_context_window", lambda model: None) + + assert rates.effective_context_window("some/unknown-model", None) == rates.DEFAULT_CONTEXT_WINDOW_TOKENS + + # --- Models whose driver would start an interactive login --- _COPILOT_MODELS = [ diff --git a/tests/test_tui_rpc_config.py b/tests/test_tui_rpc_config.py index 523ee1a8..76565944 100644 --- a/tests/test_tui_rpc_config.py +++ b/tests/test_tui_rpc_config.py @@ -161,6 +161,8 @@ async def test_config_set_model_reassigns_loop_and_persists(fake_home: Path, mon import raven.tui_rpc.methods.config as config_mod loop = _FakeLoop("old-prov", "old-model") + refreshed = [] + loop.refresh_context_window = lambda: refreshed.append(True) new_provider = SimpleNamespace(name="new-prov") monkeypatch.setattr(config_mod, "is_turn_active", lambda _key: False) @@ -188,6 +190,7 @@ async def test_config_set_model_reassigns_loop_and_persists(fake_home: Path, mon # Routed through set_provider, so everything holding the old provider # (subagents, context-engine segments, consolidator) gets told too. assert loop.switches == [(new_provider, "anthropic/claude-opus-4-8")] + assert refreshed == [True], "loop.refresh_context_window() must run after the model switch" cfg = json.loads((fake_home / ".raven" / "config.json").read_text()) assert cfg["agents"]["defaults"]["model"] == "anthropic/claude-opus-4-8" diff --git a/tests/test_tui_rpc_session_init_bundle.py b/tests/test_tui_rpc_session_init_bundle.py index 3a482a07..a56c05db 100644 --- a/tests/test_tui_rpc_session_init_bundle.py +++ b/tests/test_tui_rpc_session_init_bundle.py @@ -106,8 +106,9 @@ def fake_agent_loop_no_tracker() -> _FakeAgentLoop: @pytest.fixture() -def config(): - return load_config() +def config(tmp_path): + """Defaults only -- a real ``~/.raven/config.json`` must never leak in here.""" + return load_config(tmp_path / "does_not_exist.json") # --------------------------------------------------------------------------- @@ -144,7 +145,7 @@ def test_default_session_info_contains_real_skills(fake_agent_loop, config) -> N def test_default_session_info_contains_real_usage_baseline(fake_agent_loop, config) -> None: - """T1.1.c (AC-3): ``info.usage`` carries boot baseline (zeros + context_max from config).""" + """T1.1.c (AC-3): ``info.usage`` carries boot baseline (zeros + context_max).""" info = _default_session_info(fake_agent_loop, config) usage = info["usage"] assert isinstance(usage, dict) @@ -153,8 +154,8 @@ def test_default_session_info_contains_real_usage_baseline(fake_agent_loop, conf assert usage["output"] == 0 assert usage["cost_usd"] == 0.0 assert usage["calls"] == 0 - # context_max from config (NOT a hardcoded 200000) - assert usage["context_max"] == config.agents.defaults.context_window_tokens + # no configured pin and fake_agent_loop carries no .model -- the UI empty state + assert usage["context_max"] == 0 assert usage["context_used"] == 0 assert usage["context_percent"] == 0 @@ -197,15 +198,13 @@ def test_default_session_info_contains_real_version(fake_agent_loop, config) -> def test_context_window_reads_config_not_hardcoded_200k(fake_agent_loop, config) -> None: - """``info.context_window`` reads config, not a stub 200000.""" + """``info.context_window`` mirrors ``info.usage.context_max``, not a stub 200000.""" info = _default_session_info(fake_agent_loop, config) - assert info["context_window"] == config.agents.defaults.context_window_tokens, ( - "context_window must equal config.agents.defaults.context_window_tokens " - "(default 65536; the old stub 200000 must be gone)" - ) - # Sanity check the default is what we expect - assert config.agents.defaults.context_window_tokens == 65_536, ( - "schema default for context_window_tokens should be 65536 (schema.py:258)" + assert info["context_window"] == info["usage"]["context_max"] + assert info["context_window"] != 200_000, "the old stub 200000 must be gone" + # Sanity check the default is what we expect: None means "figure it out". + assert config.agents.defaults.context_window_tokens is None, ( + "schema default for context_window_tokens should be None (schema.py:context_window_tokens)" ) @@ -219,7 +218,7 @@ def test_default_session_info_falls_back_when_agent_loop_none(config) -> None: assert info["usage"]["input"] == 0 assert info["usage"]["output"] == 0 assert info["usage"]["calls"] == 0 - assert info["usage"]["context_max"] == config.agents.defaults.context_window_tokens + assert info["usage"]["context_max"] == 0 # version still real (importlib doesn't need agent_loop) assert info["version"] == importlib.metadata.version("raven") # lazy=True signals UI that tools/skills are placeholder (not "0 reality") @@ -239,7 +238,7 @@ def test_default_session_info_falls_back_when_no_usage_tracker(fake_agent_loop_n # usage baseline all-zero (tracker absent) assert info["usage"]["input"] == 0 assert info["usage"]["calls"] == 0 - assert info["usage"]["context_max"] == config.agents.defaults.context_window_tokens + assert info["usage"]["context_max"] == 0 # lazy=False (tools/skills are real, only usage degraded) assert info["lazy"] is False @@ -343,6 +342,19 @@ def test_boot_context_max_uses_live_window_for_openrouter(config, monkeypatch) - assert info["usage"]["context_max"] == 163840 +def test_boot_context_max_pinned_config_wins_over_live_window(config, monkeypatch) -> None: + """A pinned ``context_window_tokens`` answers even when the live window disagrees.""" + config.agents.defaults.context_window_tokens = 8192 + monkeypatch.setattr(session_module, "resolve_context_window", lambda model: 163840) + + loop = _FakeAgentLoop(with_usage_tracker=True) + loop.model = "openrouter/deepseek/deepseek-v4-pro" + + info = _default_session_info(loop, config) + + assert info["usage"]["context_max"] == 8192 + + # --------------------------------------------------------------------------- # Upgrade-nudge fields (the producer side; the TUI already reads them) # --------------------------------------------------------------------------- From 4cb44961d500be155e482996fbf537017bfa305a Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 21:08:42 +0800 Subject: [PATCH 05/78] fix(providers): recover reasoning from an orphaned closing think tag A backend launched without its reasoning parser (e.g. sglang without --reasoning-parser) swallows the opening into the prompt template, so the completion arrives as bare reasoning prose plus an orphaned . LiteLLM's only fallback anchors on an opening tag and runs non-streaming only, so the broken protocol was recorded as-is into trajectories. split_orphan_think in providers/reasoning.py splits the leading reasoning off once the full text is in hand, wired into the non-streaming parse and the stream assembly; live deltas stay untouched because a delta cannot be classified before the closing tag arrives. Co-authored-by: Claude (claude-fable-5) --- raven/agent/loop/main.py | 11 +++- raven/providers/litellm_provider.py | 5 ++ raven/providers/reasoning.py | 71 ++++++++++++++++++++++ tests/test_agent_loop_stream.py | 44 ++++++++++++++ tests/test_litellm_provider_attribution.py | 35 ++++++++++- tests/test_provider_reasoning.py | 66 ++++++++++++++++++++ 6 files changed, 229 insertions(+), 3 deletions(-) create mode 100644 raven/providers/reasoning.py create mode 100644 tests/test_provider_reasoning.py diff --git a/raven/agent/loop/main.py b/raven/agent/loop/main.py index 6cabf2c8..f99b5a31 100644 --- a/raven/agent/loop/main.py +++ b/raven/agent/loop/main.py @@ -46,6 +46,7 @@ from raven.providers.base import LLMProvider, LLMResponse, ToolCallRequest from raven.providers.capabilities import image_placeholder_text, supports_image_tool_result, vision_verdict from raven.providers.rates import effective_context_window, resolve_context_window +from raven.providers.reasoning import split_orphan_think from raven.sandbox import SandboxConfig, SandboxExecutor, SandboxInitError, build_executor from raven.session.manager import Session, SessionManager from raven.spine.turn import Origin @@ -1594,12 +1595,18 @@ async def _llm_call_stream( tool_calls = _finalize_tool_calls(tool_call_slots) finish_reason = "tool_calls" if tool_calls else "stop" + content = "".join(content_buf) + reasoning_content = "".join(reasoning_buf) or None + if reasoning_content is None: + split_reasoning, content = split_orphan_think(content) + reasoning_content = split_reasoning + return LLMResponse( - content="".join(content_buf), + content=content, tool_calls=tool_calls, finish_reason=finish_reason, usage=final_usage or {}, - reasoning_content="".join(reasoning_buf) or None, + reasoning_content=reasoning_content, ) @classmethod diff --git a/raven/providers/litellm_provider.py b/raven/providers/litellm_provider.py index f4dac57f..1aee96b8 100644 --- a/raven/providers/litellm_provider.py +++ b/raven/providers/litellm_provider.py @@ -17,6 +17,7 @@ from raven.providers.base import LLMProvider, LLMResponse, StreamDelta, ToolCallRequest from raven.providers.litellm_setup import import_litellm from raven.providers.prompt_cache import CACHE_CONTROL +from raven.providers.reasoning import split_orphan_think from raven.providers.registry import find_by_keywords, find_by_model, find_gateway from raven.providers.wire import wire_model @@ -644,6 +645,10 @@ def _parse_response(self, response: Any) -> LLMResponse: reasoning_content = getattr(message, "reasoning_content", None) or None thinking_blocks = getattr(message, "thinking_blocks", None) or None + if not reasoning_content and isinstance(content, str): + split_reasoning, content = split_orphan_think(content) + reasoning_content = split_reasoning or reasoning_content + return LLMResponse( content=content, tool_calls=tool_calls, diff --git a/raven/providers/reasoning.py b/raven/providers/reasoning.py new file mode 100644 index 00000000..caf22636 --- /dev/null +++ b/raven/providers/reasoning.py @@ -0,0 +1,71 @@ +"""Recovering reasoning text a backend meant to hide but couldn't. + +Prompt caching answers "does this request understand the field" as +(wire x model family) -- two axes, both readable from the request. Reasoning +tagging has a third axis neither of those covers: whether the *inference +server* was started with a parser for the model's think tags. sglang and +vLLM both ship one, but it is an opt-in flag; run without it and the tag the +model was trained to emit at the start of its turn is swallowed into the +prompt template before generation even begins, so the completion that comes +back is not a well-formed ``...`` block but its second half: +bare reasoning prose followed by an orphaned ````, with no opening +tag anywhere in the text to pair it with. + +LiteLLM has one fallback for this, ``_parse_content_for_reasoning`` in +``litellm_core_utils/prompt_templates/common_utils.py``, but it anchors on +the tag at the *start* of the string with ``re.match`` -- built for a model +that emits ``reasoninganswer`` and drops only the wrapper, not +for a string that already begins mid-reasoning. It also runs on the +non-streaming path only. So this is where the missing half of that fallback +lives: given the shape sglang/vLLM without ``--reasoning-parser`` actually +produce, split the leading reasoning off from the answer that follows the +orphaned closing tag. + +This module only ever sees text that is already whole. A streaming delta +cannot be judged as it arrives -- until the closing tag shows up there is no +way to tell "the model is still reasoning" apart from "the model already +started its answer and just writes like that", and buffering deltas on +spec to find out would turn a streaming response into a delayed one for +every model that behaves normally. So there is no delta-level counterpart +here; callers normalize once the full text is in hand, whether that text +came back in one response or was assembled from a stream. +""" + +from __future__ import annotations + +import re + +_CLOSE_TAG_RE = re.compile(r"|", re.IGNORECASE) + + +def split_orphan_think(text: str) -> tuple[str | None, str]: + """Split ``text`` into ``(reasoning, content)`` if it holds an orphan closing tag. + + An orphan is a ```` or ```` with no matching opening tag + before it -- the shape produced when the server swallowed the opener into + its prompt template. A paired block (opener present earlier in the text) + is left alone and returned as ``(None, text)`` unchanged, for the existing + complete-block handlers (``_strip_think`` et al.) to take care of; so is + text with no closing tag at all. + + Everything before the tag becomes ``reasoning`` once stripped, unless that + strips to nothing, in which case ``reasoning`` is ``None`` and only the + stray tag is removed. Everything after the tag becomes ``content``, minus + a single leading newline (the one the model wrote to separate reasoning + from answer, not meaningful content). + """ + match = _CLOSE_TAG_RE.search(text) + if match is None: + return None, text + + open_tag = "" if match.group(0).lower() == "" else "" + prefix = text[: match.start()] + if open_tag in prefix.lower(): + return None, text + + rest = text[match.end() :] + if rest.startswith("\n"): + rest = rest[1:] + + reasoning = prefix.strip() + return (reasoning or None), rest diff --git a/tests/test_agent_loop_stream.py b/tests/test_agent_loop_stream.py index 28e7501c..fc30c102 100644 --- a/tests/test_agent_loop_stream.py +++ b/tests/test_agent_loop_stream.py @@ -259,3 +259,47 @@ async def on_delta(_text: str) -> None: assert response.content == "" assert response.tool_calls == [] assert response.finish_reason == "stop" + + +# --------------------------------------------------------------------------- +# Orphan recovery (issue #152) -- backend never emitted a structured +# reasoning delta, and the accumulated content carries a closing tag with no +# opener (the server's prompt template swallowed it). +# --------------------------------------------------------------------------- + + +async def test_llm_call_stream_splits_orphan_think_from_content() -> None: + chunks = [ + StreamDelta(content="raw reasoning"), + StreamDelta(content="\n"), + StreamDelta(content="final answer"), + ] + provider = _FakeProvider(chunks) + call = _bind_helper(provider) + + async def on_delta(_text: str) -> None: + return None + + response = await call(messages=[], tools=None, model="m", on_token_delta=on_delta) + + assert response.reasoning_content == "raw reasoning" + assert response.content == "final answer" + + +async def test_llm_call_stream_leaves_structured_reasoning_alone() -> None: + """A non-empty structured reasoning_content stream wins outright; an + orphan tag inside content (if any) is left untouched.""" + chunks = [ + StreamDelta(content=None, reasoning_content="thinking"), + StreamDelta(content="visible more text"), + ] + provider = _FakeProvider(chunks) + call = _bind_helper(provider) + + async def on_delta(_text: str) -> None: + return None + + response = await call(messages=[], tools=None, model="m", on_token_delta=on_delta) + + assert response.reasoning_content == "thinking" + assert response.content == "visible more text" diff --git a/tests/test_litellm_provider_attribution.py b/tests/test_litellm_provider_attribution.py index e8dc14cb..2cca1841 100644 --- a/tests/test_litellm_provider_attribution.py +++ b/tests/test_litellm_provider_attribution.py @@ -2,7 +2,7 @@ from __future__ import annotations -from unittest.mock import patch +from unittest.mock import MagicMock, patch from raven.providers.litellm_provider import _ANTHROPIC_EXTRA_KEYS, LiteLLMProvider @@ -62,3 +62,36 @@ def test_extra_msg_keys_matches_on_resolved_anthropic_prefix(): def test_extra_msg_keys_non_anthropic_preserves_nothing(): assert LiteLLMProvider._extra_msg_keys("gpt-4o", "gpt-4o") == frozenset() + + +# --- orphan recovery in _parse_response (issue #152, keyless, no live call) --- +# A backend run without a reasoning parser swallows the opening tag into its +# prompt template and returns bare reasoning text + a lone ``. Covers +# both directions: the split fires when there is no structured +# reasoning_content, and stays out of the way when there is one. + + +def _fake_response(content: str, reasoning_content: str | None = None) -> MagicMock: + message = MagicMock(content=content, tool_calls=None, reasoning_content=reasoning_content, thinking_blocks=None) + choice = MagicMock(message=message, finish_reason="stop") + return MagicMock(choices=[choice], usage=None) + + +def test_parse_response_splits_orphan_think_into_reasoning(): + provider = _make_provider("openai") + response = _fake_response("raw reasoning text\nfinal answer") + + result = provider._parse_response(response) + + assert result.reasoning_content == "raw reasoning text" + assert result.content == "final answer" + + +def test_parse_response_leaves_structured_reasoning_content_alone(): + provider = _make_provider("openai") + response = _fake_response("visible\nanswer", reasoning_content="already structured") + + result = provider._parse_response(response) + + assert result.reasoning_content == "already structured" + assert result.content == "visible\nanswer" diff --git a/tests/test_provider_reasoning.py b/tests/test_provider_reasoning.py new file mode 100644 index 00000000..89ee0c54 --- /dev/null +++ b/tests/test_provider_reasoning.py @@ -0,0 +1,66 @@ +"""Tests for raven.providers.reasoning -- recovering an orphaned . + +A backend run without a reasoning parser swallows the opening tag into its +prompt template, so the completion carries bare reasoning prose followed by a +lone closing tag and no opener to pair it with. ``split_orphan_think`` is the +one place that shape gets recognized and split; a paired block or no tag at +all must pass through untouched for the existing complete-block handlers. +""" + +from __future__ import annotations + +from raven.providers.reasoning import split_orphan_think + + +def test_orphan_close_tag_splits_reasoning_from_content(): + reasoning, content = split_orphan_think("raw reasoning text\nfinal answer") + + assert reasoning == "raw reasoning text" + assert content == "final answer" + + +def test_paired_complete_block_is_left_untouched(): + text = "raw reasoning\nfinal answer" + + assert split_orphan_think(text) == (None, text) + + +def test_no_closing_tag_is_left_untouched(): + text = "just a plain answer, no tags at all" + + assert split_orphan_think(text) == (None, text) + + +def test_orphan_thinking_variant_splits_too(): + reasoning, content = split_orphan_think("some reasoning\nthe answer") + + assert reasoning == "some reasoning" + assert content == "the answer" + + +def test_whitespace_only_prefix_strips_the_tag_without_a_reasoning_value(): + reasoning, content = split_orphan_think(" \n\nthe answer") + + assert reasoning is None + assert content == "the answer" + + +def test_content_after_the_tag_is_preserved_beyond_the_first_newline(): + reasoning, content = split_orphan_think("reasoning\nline one\nline two") + + assert reasoning == "reasoning" + assert content == "line one\nline two" + + +def test_only_a_single_leading_newline_is_dropped_from_content(): + reasoning, content = split_orphan_think("reasoning\n\nblank line kept") + + assert reasoning == "reasoning" + assert content == "\nblank line kept" + + +def test_content_without_a_leading_newline_is_unchanged(): + reasoning, content = split_orphan_think("reasoningimmediate answer") + + assert reasoning == "reasoning" + assert content == "immediate answer" From 79f5d96e6464a5a967bb162c4481efe1b25e4a85 Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 21:15:34 +0800 Subject: [PATCH 06/78] fix(*): refuse wizard vendors a bare api key cannot configure The onboarding wizard's "another supported vendor" branch treated every LiteLLM vendor as a paste-a-key shape, so chatgpt (device flow that ignores the key), bedrock/sagemaker (AWS credential chain), vertex_ai (project+location+ADC), azure (base+version+key/Entra) and cloudflare (key+base/account_id) were written down as configured and then failed on the first call. auth.py now carries the refusal table with what each vendor actually needs, and _collect_credentials refuses before collecting instead of persisting a section that can never authenticate. chatgpt points at raven's own openai_codex flow; gigachat stays configurable and only gains a hint that its key is base64(client_id:client_secret). Deliberately no registry specs or multi-credential schema for these six until someone actually needs them served. Co-authored-by: Claude (claude-fable-5) --- raven/cli/onboard_commands.py | 25 +++++++- raven/providers/auth.py | 67 +++++++++++++++++++++ tests/test_cli_onboard_commands.py | 97 ++++++++++++++++++++++++++++++ tests/test_provider_auth_method.py | 33 ++++++++++ 4 files changed, 221 insertions(+), 1 deletion(-) diff --git a/raven/cli/onboard_commands.py b/raven/cli/onboard_commands.py index 28d21884..a6d3701a 100644 --- a/raven/cli/onboard_commands.py +++ b/raven/cli/onboard_commands.py @@ -1426,7 +1426,17 @@ def _collect_credentials( """Auth setup: OAuth browser flow or api_key write. Returns the custom model id when the provider is ``custom`` (locked in here), ``None`` for a non-custom provider, or ``_BACK`` if the user backed out of the first - interactive credential field (caller should rewind to the picker).""" + interactive credential field, or if the vendor cannot be configured by a + bare key at all (caller should rewind to the picker either way).""" + from raven.providers.auth import key_refusal + + refusal = key_refusal(provider) + if refusal is not None: + console.print(f" [red]x[/red] {refusal}") + if non_interactive: + raise typer.Exit(2) + return _BACK + if is_oauth: if non_interactive: console.print( @@ -1483,6 +1493,19 @@ def _collect_credentials( _write_provider_fields(provider, {"api_base": base_url}) return None + if not api_key: + from raven.providers.registry import normalize_provider_name + + # GigaChat's key is not a typical API key -- it is base64(client_id: + # client_secret) -- and the generic prompt below gives no room to say + # so, so the wizard would otherwise send someone looking for a plain + # key straight into a 401. + if normalize_provider_name(provider) == "gigachat": + console.print( + " [dim]GigaChat's key is base64(client_id:client_secret) from the " + "GigaChat API console, not a typical API key.[/dim]" + ) + # Pure interactive path (no creds came from flags): prompt field-by-field # with empty-submit = back; backing out of the first field rewinds to the # provider picker. diff --git a/raven/providers/auth.py b/raven/providers/auth.py index 6132c223..14e84dcd 100644 --- a/raven/providers/auth.py +++ b/raven/providers/auth.py @@ -263,6 +263,73 @@ def credential_status( return CredentialStatus(name, False, method.kind, gap, method) +#: Vendors LiteLLM reaches by API key, that Raven carries no spec for, where a +#: bare key still cannot configure them -- the credential shape needs more than +#: the wizard's generic "paste a key" branch offers. Every other unspecced +#: vendor is in fact reached by a single key; this only lists the ones that +#: are not, so ``key_refusal`` can say what actually gets each one working +#: instead of the wizard writing down a key that 401s (or, for chatgpt, is +#: silently ignored) at the first call. +#: +#: Deliberately not registry entries: a spec exists to drive routing (default +#: model, client selection, the connectivity probe), and none of these six get +#: any of that from Raven -- adding a spec just to hold a rejection message +#: would build the scaffolding this module exists to avoid. +_KEY_CANNOT_CONFIGURE: dict[str, str] = { + "chatgpt": ( + "ChatGPT is reached through its own OAuth device flow -- LiteLLM's " + "chatgpt transformation ignores any api_key and authenticates through a " + "stored browser session instead. Raven already has this path: run " + "`raven provider login openai-codex` (or pick \"OpenAI Codex (OAuth)\" " + "from this menu)." + ), + "bedrock": ( + "Bedrock is reached through the AWS credential chain -- an access key " + "and secret plus a region, an AWS profile, or ambient credentials from " + "the environment or instance role -- not a single api_key field. Set " + "AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_REGION (or an AWS " + "profile) in the environment instead." + ), + "sagemaker": ( + "SageMaker is reached through the same AWS credential chain as " + "Bedrock -- an access key and secret plus a region, an AWS profile, or " + "ambient credentials -- not a single api_key field. Set " + "AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_REGION (or an AWS " + "profile) in the environment instead." + ), + "vertex_ai": ( + "Vertex AI needs a project and a location, plus either a service " + "account credentials JSON or ambient Application Default Credentials -- " + "not a single api_key field. Set VERTEXAI_PROJECT / VERTEXAI_LOCATION " + "and either GOOGLE_APPLICATION_CREDENTIALS or run `gcloud auth " + "application-default login`." + ), + "azure": ( + "This is LiteLLM's native Azure vendor, which needs an api_base and an " + "api_version plus either an api_key or Entra ID auth -- a bare key is " + "not enough. Raven's own Azure path (`azure_openai`) already asks for " + "the base URL and version; pick that instead." + ), + "cloudflare": ( + "Cloudflare Workers AI needs an api_key plus either an api_base or an " + "account_id -- a bare key alone is not enough. Configure --base-url " + "(or an account id) alongside the key." + ), +} + + +def key_refusal(vendor: str) -> str | None: + """Why a bare API key cannot configure this vendor, or ``None`` if a key works. + + Checked before the onboarding wizard's generic key-only branch writes a + config section for a vendor Raven carries no spec for, so it can refuse + with the reason rather than persist a key that will never authenticate. + """ + from raven.providers.registry import normalize_provider_name + + return _KEY_CANNOT_CONFIGURE.get(normalize_provider_name(vendor)) + + def _localize(req: Requirement, name: str) -> Requirement: """Put the provider's own name into the hint, so it can be pasted. diff --git a/tests/test_cli_onboard_commands.py b/tests/test_cli_onboard_commands.py index 4fb005d2..d2c0b768 100644 --- a/tests/test_cli_onboard_commands.py +++ b/tests/test_cli_onboard_commands.py @@ -402,6 +402,36 @@ def test_onboard_oauth_non_interactive_errors(tmp_env: Path) -> None: assert "OAuth providers require an interactive browser flow" in r.stdout +@pytest.mark.parametrize("vendor", ["chatgpt", "bedrock", "sagemaker", "vertex_ai", "azure", "cloudflare"]) +def test_onboard_non_interactive_bare_key_refused_vendor_errors(tmp_env: Path, vendor: str) -> None: + """A vendor issue #254 identified as unconfigurable by a bare key is + refused before any credentials are written, instead of being sent through + the generic single-key branch that would 401 (or, for chatgpt, be + silently ignored) at the first call.""" + r = runner.invoke( + app, + [ + "onboard", + "--non-interactive", + "--provider", + vendor, + "--api-key", + "sk-fake", + "--skip-channel", + "--yes", + ], + ) + assert r.exit_code != 0 + from raven.providers.auth import key_refusal + + reason = key_refusal(vendor) + assert reason is not None + out = " ".join(r.stdout.split()) + assert " ".join(reason.split()) in out + data = json.loads(tmp_env.read_text()) + assert vendor not in data.get("providers", {}) + + def test_onboard_non_tty_no_flag_fails(tmp_env: Path) -> None: """Without a TTY and without ``--non-interactive`` we give a clear hint. @@ -1867,6 +1897,73 @@ def _verify(name, *a, **kw): assert data["agents"]["defaults"]["model"] == "openai/gpt-5.5" +def test_step1_bare_key_refused_vendor_rewinds_to_picker( + tmp_env: Path, monkeypatch: pytest.MonkeyPatch, stub_verify, stub_step3, capsys: pytest.CaptureFixture +) -> None: + """Picking a vendor issue #254 identified as unconfigurable by a bare key + (chatgpt: it authenticates through Raven's own OAuth path instead) prints + the reason and rewinds to the picker via the wizard's existing back + mechanism, the same one 'Switch provider' uses -- instead of prompting for + a key that would never authenticate. + """ + picks = iter(["chatgpt", "openai"]) + key_prompts: list[str] = [] + monkeypatch.setattr(onboard_commands, "_check_tty_or_die", lambda non_interactive: None) + monkeypatch.setattr(onboard_commands, "_pick_language", lambda: None) + monkeypatch.setattr(onboard_commands, "_select_provider", lambda: next(picks)) + + def _fake_prompt_api_key(provider, **kw): + key_prompts.append(provider) + return f"sk-{provider}" + + monkeypatch.setattr(onboard_commands, "_prompt_api_key", _fake_prompt_api_key) + monkeypatch.setattr(onboard_commands, "_pick_model", lambda provider, spec, **_: spec.default_model) + monkeypatch.setattr(onboard_commands, "_step2_sandbox", lambda **_: None) + monkeypatch.setattr(onboard_commands, "_step3_channel", lambda **_: None) + monkeypatch.setattr(onboard_commands, "_step4_memory", lambda **_: None) + monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) + monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) + + onboard_commands.run_wizard(non_interactive=False) + + out = " ".join(capsys.readouterr().out.split()) + from raven.providers.auth import key_refusal + + assert " ".join(key_refusal("chatgpt").split()) in out + # The refused vendor never reached the key prompt at all. + assert key_prompts == ["openai"] + data = json.loads(tmp_env.read_text()) + assert "chatgpt" not in data.get("providers", {}) + assert data["providers"]["openai"]["apiKey"] == "sk-openai" + assert data["agents"]["defaults"]["model"] == "openai/gpt-5.5" + + +def test_collect_credentials_gigachat_hints_key_shape_before_prompting( + tmp_env: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + """GigaChat *can* be configured by a bare key -- it's just an unusual one + (base64(client_id:client_secret)) -- so the wizard hints at its shape + instead of refusing it.""" + monkeypatch.setattr(onboard_commands, "_prompt_api_key", lambda provider, **kw: "Z2lnYWNoYXQ6c2VjcmV0") + + result = onboard_commands._collect_credentials( + "gigachat", + is_oauth=False, + is_custom=False, + is_local=False, + api_key=None, + base_url=None, + model=None, + non_interactive=False, + ) + + assert result is None + out = " ".join(capsys.readouterr().out.split()) + assert "base64(client_id:client_secret)" in out + data = json.loads(tmp_env.read_text()) + assert data["providers"]["gigachat"]["apiKey"] == "Z2lnYWNoYXQ6c2VjcmV0" + + def test_add_provider_keeps_existing(tmp_env: Path, monkeypatch: pytest.MonkeyPatch, stub_verify, stub_step3) -> None: """Adding a second provider in the existing-config entry doesn't drop the first.""" _seed_provider("openai", "sk-first", "openai/gpt-4o-mini") diff --git a/tests/test_provider_auth_method.py b/tests/test_provider_auth_method.py index 891e94e2..32f8ed5a 100644 --- a/tests/test_provider_auth_method.py +++ b/tests/test_provider_auth_method.py @@ -32,6 +32,7 @@ import pytest from raven.config.schema import Config +from raven.providers.auth import key_refusal #: Provider sections paired with the model id that selects them. Each case is a #: shape of credential material, not a vendor: "the key is in the plural field", @@ -299,3 +300,35 @@ def key_reads(tree: ast.AST) -> list[int]: for line in key_reads(ast.parse(path.read_text())) ) assert not offenders, "decide configuredness through providers.auth.credential_status: " + ", ".join(offenders) + + +#: The six vendors issue #254 identified as unconfigurable by a bare key -- +#: each needs credential material the onboarding wizard's generic single-key +#: prompt has no field for. +_KEY_REFUSED_VENDORS = ("chatgpt", "bedrock", "sagemaker", "vertex_ai", "azure", "cloudflare") + + +@pytest.mark.parametrize("vendor", _KEY_REFUSED_VENDORS) +def test_key_refusal_names_a_reason_for_vendors_a_key_cannot_configure(vendor: str) -> None: + reason = key_refusal(vendor) + assert reason is not None + assert reason.strip() + + +def test_key_refusal_chatgpt_points_at_ravens_own_oauth_path() -> None: + """chatgpt is the one vendor where a *different* Raven path already exists.""" + reason = key_refusal("chatgpt") + assert reason is not None + assert "openai-codex" in reason or "openai_codex" in reason + + +@pytest.mark.parametrize("vendor", ["gigachat", "openai", "anthropic", "custom", "deepseek"]) +def test_key_refusal_is_none_for_vendors_a_key_configures(vendor: str) -> None: + """Everyone else -- including gigachat, whose key merely has an odd shape.""" + assert key_refusal(vendor) is None + + +def test_key_refusal_normalizes_hyphen_and_case() -> None: + """Matched the same way every other provider-name comparison is made.""" + assert key_refusal("Vertex-AI") == key_refusal("vertex_ai") + assert key_refusal("BEDROCK") == key_refusal("bedrock") From cc481b9780f8055ec3978a2785f72d4d92258823 Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 21:17:45 +0800 Subject: [PATCH 07/78] test(providers): pin api_key forwarding into acompletion kwargs Investigating a reported spawn-subagent 401 established that chat() and chat_stream() both pass the provider's api_key explicitly to acompletion and that a subagent reuses the main provider instance verbatim -- the reported asymmetry is not reproducible from the code, on HEAD or on the release it was reported against. What the investigation did find is that no test ever asserted the api_key kwarg arrives, so the explicit-forwarding line could be dropped without anything turning red. These three tests pin the mechanism on both call paths and through the subagent manager's shared instance. Co-authored-by: Claude (claude-fable-5) --- tests/test_litellm_provider_stream.py | 63 +++++++++++++++++++++++++++ tests/test_subagent_manager.py | 52 ++++++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/tests/test_litellm_provider_stream.py b/tests/test_litellm_provider_stream.py index d8d4e7f9..b6d67217 100644 --- a/tests/test_litellm_provider_stream.py +++ b/tests/test_litellm_provider_stream.py @@ -6,6 +6,9 @@ - None-content chunks (e.g. final stop chunk) are skipped (return None → no yield) - signature parity with chat() (messages/tools/model/max_tokens/temperature/ reasoning_effort/tool_choice all accepted; stream=True forwarded to acompletion) +- chat() and chat_stream() both forward the provider's api_key to acompletion + as an explicit kwarg, rather than relying on it having been exported to the + environment Mocks patch `raven.providers.litellm_provider.acompletion` because the provider module imports `from litellm import acompletion` at top level, so @@ -48,6 +51,14 @@ def _chunk(content: str | None) -> _FakeChunk: return _FakeChunk(choices=[_FakeChoice(delta=_FakeDelta(content=content))]) +class _FakeResponse: + """Non-streaming acompletion result with one text choice.""" + + def __init__(self, text: str) -> None: + self.choices = [_FakeChoice(delta=_FakeDelta(content=text), finish_reason="stop")] + self.usage = None + + async def _fake_stream(chunks: list[_FakeChunk]): """Async generator standing in for litellm's streamed response.""" for ch in chunks: @@ -180,3 +191,55 @@ async def fake_acompletion(**kwargs: Any): assert captured["tools"] == tools # model should be resolved (openai/gpt-4o-mini already has prefix → stays the same) assert "gpt-4o-mini" in captured["model"] + + +@pytest.mark.asyncio +async def test_chat_forwards_api_key_to_acompletion(monkeypatch: pytest.MonkeyPatch) -> None: + """chat() must pass the provider's api_key explicitly to acompletion. + + A subagent spawned in-process reuses the main provider instance (see + SubagentManager), so if this explicit forwarding were ever dropped in + favor of relying on an exported environment variable, a request made + under a different/missing env context (e.g. a subprocess or a provider + with no matching env var) would silently lose the key. + """ + captured: dict[str, Any] = {} + + async def fake_acompletion(**kwargs: Any): + captured.update(kwargs) + return _FakeResponse("hi") + + monkeypatch.setattr( + "raven.providers.litellm_provider.acompletion", + fake_acompletion, + ) + + provider = LiteLLMProvider(api_key="k-main", default_model="openai/gpt-4o") + await provider.chat(messages=[{"role": "user", "content": "hi"}], model="openai/gpt-4o") + + assert captured["api_key"] == "k-main" + + +@pytest.mark.asyncio +async def test_chat_stream_forwards_api_key_to_acompletion(monkeypatch: pytest.MonkeyPatch) -> None: + """chat_stream() must pass the provider's api_key explicitly to acompletion. + + Same regression as test_chat_forwards_api_key_to_acompletion, for the + streaming code path. + """ + captured: dict[str, Any] = {} + + async def fake_acompletion(**kwargs: Any): + captured.update(kwargs) + return _fake_stream([_chunk("ok")]) + + monkeypatch.setattr( + "raven.providers.litellm_provider.acompletion", + fake_acompletion, + ) + + provider = LiteLLMProvider(api_key="k-main", default_model="openai/gpt-4o") + async for _ in provider.chat_stream(messages=[{"role": "user", "content": "hi"}]): + pass + + assert captured["api_key"] == "k-main" diff --git a/tests/test_subagent_manager.py b/tests/test_subagent_manager.py index 15298c3f..073d13b0 100644 --- a/tests/test_subagent_manager.py +++ b/tests/test_subagent_manager.py @@ -4,12 +4,19 @@ test drives only the Semaphore in _run_subagent (no real VM, no real LLM). A stubbed inner holds each subagent inside the gate on an Event, letting the test observe the concurrent peak. + +Also covers: a subagent reuses the main LiteLLMProvider instance verbatim +(SubagentManager.provider), so the api_key that instance was constructed +with reaches acompletion on the subagent's own chat_with_retry() calls too — +acompletion is mocked, so this stays "no real LLM". """ from __future__ import annotations import asyncio from pathlib import Path +from types import SimpleNamespace +from typing import Any import pytest from pydantic import ValidationError @@ -18,6 +25,7 @@ from raven.agent.subagent.manager import SubagentManager from raven.config.schema import AgentDefaults from raven.providers.base import LLMResponse, ToolCallRequest +from raven.providers.litellm_provider import LiteLLMProvider from raven.sandbox import ExecResult, SandboxExecutor @@ -327,3 +335,47 @@ def _spy_init(self, workspace, *args, **kwargs): mgr._build_subagent_prompt() assert calls == [False] + + +async def test_subagent_reuses_main_provider_and_forwards_api_key(monkeypatch): + """A subagent runs in-process against the exact provider instance the main + agent was built with (manager.provider), not a fresh one — so an api_key + set only on the main instance must still reach acompletion for the + subagent's own chat_with_retry() calls. + + A reported spawn-subagent 401 could not be reproduced by reading the + code (chat()/chat_stream() already pass api_key explicitly to + acompletion), but nothing in the test suite actually asserted that + kwarg ever arrived -- this pins it down. + """ + captured: dict[str, Any] = {} + + async def fake_acompletion(**kwargs: Any): + captured.update(kwargs) + return SimpleNamespace( + choices=[ + SimpleNamespace( + message=SimpleNamespace(content="ok", tool_calls=None), + finish_reason="stop", + ) + ], + usage=None, + ) + + monkeypatch.setattr( + "raven.providers.litellm_provider.acompletion", + fake_acompletion, + ) + + provider = LiteLLMProvider(api_key="k-main", default_model="openai/gpt-4o") + manager = SubagentManager(provider=provider, workspace=Path("/tmp")) + + assert manager.provider is provider + + response = await manager.provider.chat_with_retry( + messages=[{"role": "user", "content": "hi"}], + model="openai/gpt-4o", + ) + + assert response.finish_reason != "error" + assert captured["api_key"] == "k-main" From ea9fdd296d606c8923ac1f2714bc4fa375956da3 Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 21:24:54 +0800 Subject: [PATCH 08/78] fix(providers): merge user extra_body with wire routing instead of overwriting model_overrides is already the channel for arbitrary sampling/serving params -- LiteLLM auto-forwards unknown top-level keys into extra_body for OpenAI-compatible backends, and a nested structure can be written as extra_body directly. But both chat() and chat_stream() assigned the provider's wire-routing extra_body (e.g. the OpenRouter provider pin) over whatever a model_overrides entry had placed there, so a user's extra_body keys were silently dropped whenever the request travelled through a gateway. Merge the two instead, wire keys winning on collision because routing pins must reach the wire intact. The schema comment now names model_overrides as the passthrough channel, so no sampling_extra/request_extra fields get invented for the same decision. Co-authored-by: Claude (claude-fable-5) --- raven/config/schema.py | 4 ++ raven/providers/litellm_provider.py | 21 +++++- .../test_litellm_provider_model_overrides.py | 70 +++++++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/raven/config/schema.py b/raven/config/schema.py index 2e5dca38..9bffb253 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -283,6 +283,10 @@ class AgentDefaults(Base): # name: {"kimi-k2.5": {"temperature": 1.0}}. Some models reject the usual # defaults, and hard-coding those quirks in the registry left users unable to # adjust them. Entries here win over the registry's built-in defaults. + # This is also the direct channel for arbitrary sampling/serving params: an + # unknown top-level key is auto-forwarded into extra_body by LiteLLM for + # OpenAI-compatible backends (e.g. sglang's repetition_penalty); a nested + # structure can be written directly as extra_body: {...}. model_overrides: dict[str, dict[str, Any]] = Field(default_factory=dict) enable_personalization: bool = False # 4-step PAHF-inspired personalization flow (classify → ask → execute → learn) diff --git a/raven/providers/litellm_provider.py b/raven/providers/litellm_provider.py index 1aee96b8..ce115fe3 100644 --- a/raven/providers/litellm_provider.py +++ b/raven/providers/litellm_provider.py @@ -60,6 +60,23 @@ def _short_tool_id() -> str: return "".join(secrets.choice(_ALNUM) for _ in range(9)) +def _merge_extra_body(kwargs: dict[str, Any], wire_extra_body: dict[str, Any]) -> None: + """Merge the provider's wire-routing extra_body into kwargs instead of overwriting it. + + A model_overrides entry (see _apply_model_overrides) may have already placed + a user extra_body dict in kwargs -- for example Qwen3's + extra_body.chat_template_kwargs.enable_thinking. Assigning wire_extra_body + over it would silently drop those keys. On a key collision, wire_extra_body + wins: it carries routing pins (e.g. OpenRouter provider order) that must + reach the wire intact. + """ + existing = kwargs.get("extra_body") + if isinstance(existing, dict): + kwargs["extra_body"] = {**existing, **wire_extra_body} + else: + kwargs["extra_body"] = wire_extra_body + + def session_affinity_headers() -> dict[str, str]: """Headers pinning one caller to one backend replica. @@ -336,7 +353,7 @@ async def chat( # Pass provider-specific body extras (e.g. OpenRouter routing pin) if self.extra_body: - kwargs["extra_body"] = self.extra_body + _merge_extra_body(kwargs, self.extra_body) if reasoning_effort: kwargs["reasoning_effort"] = reasoning_effort @@ -414,7 +431,7 @@ async def chat_stream( if self.extra_headers: kwargs["extra_headers"] = self.extra_headers if self.extra_body: - kwargs["extra_body"] = self.extra_body + _merge_extra_body(kwargs, self.extra_body) if reasoning_effort: kwargs["reasoning_effort"] = reasoning_effort kwargs["drop_params"] = True diff --git a/tests/test_litellm_provider_model_overrides.py b/tests/test_litellm_provider_model_overrides.py index ad83e201..cb1d50c2 100644 --- a/tests/test_litellm_provider_model_overrides.py +++ b/tests/test_litellm_provider_model_overrides.py @@ -133,3 +133,73 @@ async def test_config_override_wins_over_registry(monkeypatch: pytest.MonkeyPatc await p.chat(messages=[{"role": "user", "content": "hi"}], temperature=0.1) assert seen[0]["temperature"] == 0.5 + + +@pytest.mark.asyncio +async def test_model_override_forwards_arbitrary_backend_param(monkeypatch: pytest.MonkeyPatch) -> None: + # sglang's repetition_penalty has no dedicated kwarg in chat() -- it must + # reach LiteLLM as a top-level kwarg so LiteLLM auto-forwards it into + # extra_body for OpenAI-compatible backends. + seen = _capture(monkeypatch) + p = _provider("my-sglang-model", {"my-sglang-model": {"repetition_penalty": 1.05}}) + + await p.chat(messages=[{"role": "user", "content": "hi"}]) + + assert seen[0]["repetition_penalty"] == 1.05 + + +@pytest.mark.asyncio +async def test_model_override_extra_body_merges_with_wire_routing(monkeypatch: pytest.MonkeyPatch) -> None: + # The user's Qwen3 chat_template_kwargs must survive alongside the + # provider's own OpenRouter routing pin, not be clobbered by it. + seen = _capture(monkeypatch) + p = LiteLLMProvider( + api_key="test-key", + default_model="openrouter/qwen/qwen3", + provider_name="openrouter", + extra_body={"provider": {"order": ["Alibaba"]}}, + model_overrides={"qwen3": {"extra_body": {"chat_template_kwargs": {"enable_thinking": False}}}}, + ) + p.generation = GenerationSettings(temperature=0.1) + + await p.chat(messages=[{"role": "user", "content": "hi"}]) + + assert seen[0]["extra_body"] == { + "chat_template_kwargs": {"enable_thinking": False}, + "provider": {"order": ["Alibaba"]}, + } + + +@pytest.mark.asyncio +async def test_model_override_extra_body_wire_key_wins_on_conflict(monkeypatch: pytest.MonkeyPatch) -> None: + # On a colliding key, the provider's own wire-routing extra_body must win -- + # routing correctness over a user override that would misroute the request. + seen = _capture(monkeypatch) + p = LiteLLMProvider( + api_key="test-key", + default_model="openrouter/qwen/qwen3", + provider_name="openrouter", + extra_body={"provider": {"order": ["Anthropic"]}}, + model_overrides={"qwen3": {"extra_body": {"provider": {"order": ["Alibaba"]}}}}, + ) + p.generation = GenerationSettings(temperature=0.1) + + await p.chat(messages=[{"role": "user", "content": "hi"}]) + + assert seen[0]["extra_body"] == {"provider": {"order": ["Anthropic"]}} + + +@pytest.mark.asyncio +async def test_extra_body_unchanged_without_user_override(monkeypatch: pytest.MonkeyPatch) -> None: + seen = _capture(monkeypatch) + p = LiteLLMProvider( + api_key="test-key", + default_model="openrouter/moonshotai/kimi-k2.5", + provider_name="openrouter", + extra_body={"provider": {"order": ["Anthropic"]}}, + ) + p.generation = GenerationSettings(temperature=0.1) + + await p.chat(messages=[{"role": "user", "content": "hi"}]) + + assert seen[0]["extra_body"] == {"provider": {"order": ["Anthropic"]}} From 70b92f6f16dde6d186b529d047f8ebdf04a72c25 Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 21:27:23 +0800 Subject: [PATCH 09/78] docs(readme): invite vendor partnerships in the contributing section The picker already marks MiniMax as an open-source partner; this adds the one sentence telling other vendors that the door is open. Co-authored-by: Claude (claude-fable-5) --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 75578672..de9dde68 100644 --- a/README.md +++ b/README.md @@ -503,6 +503,9 @@ Raven is early, and useful contributions are welcome across runtime architecture, TUI polish, provider support, memory workflows, proactivity, benchmarks, documentation, and issue reports. +Model vendors interested in a first-party integration or an open-source +partnership (as MiniMax already has) are welcome to open an issue and say so. + Before opening a PR: 1. Read [AGENTS.md](AGENTS.md). From 336eb8ff60625f354e24315add0169b4484cc9df Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 21:32:32 +0800 Subject: [PATCH 10/78] test(cli): pin picker-registry parity and wizard prefill defaults Two guards the wizard was missing. The curated picker catalogue is hand-written and the existing tests only checked one direction, so a provider added to the registry but never to the shortlist stayed unreachable with everything green; the new set-equality test turns either drift direction red. And both prefill paths -- the language screen defaulting to the active language, and the model prompt defaulting to the already-configured model -- had no assertions at all (every _pick_model test passed current_model=None), so deleting either default left the suite green. Co-authored-by: Claude (claude-fable-5) --- tests/test_cli_onboard_commands.py | 85 ++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/tests/test_cli_onboard_commands.py b/tests/test_cli_onboard_commands.py index d2c0b768..150dd472 100644 --- a/tests/test_cli_onboard_commands.py +++ b/tests/test_cli_onboard_commands.py @@ -233,6 +233,51 @@ def test_curated_providers_do_not_restate_registry_flags() -> None: assert "is_oauth" not in entry +def test_curated_and_registry_provider_names_match_exactly() -> None: + """The curated catalogue must name exactly the registry's providers, no more + and no fewer. + + ``test_curated_providers_all_exist_in_registry`` only checks one direction + (nothing curated is unknown to the registry) -- a provider added to the + registry and never added to this hand-written shortlist passed that test + silently, and stayed unreachable from the wizard's picker. Comparing the + full sets both ways means either mistake, in either direction, turns this + test red. The sentinel row is not a provider, so it is added to the + registry side rather than dropped from the curated one. + """ + from raven.providers.registry import PROVIDERS + + curated_names = {entry["name"] for group in onboard_commands._CURATED_GROUPS for entry in group["providers"]} + registry_names = {spec.name for spec in PROVIDERS} + assert curated_names == registry_names | {onboard_commands._PICK_LITELLM_VENDOR} + + +# --------------------------------------------------------------------------- language step + + +def test_pick_language_preselects_the_currently_active_language(monkeypatch: pytest.MonkeyPatch) -> None: + """A re-run of the wizard must default the language screen to whatever + language is already active, not silently reset a Chinese user to English. + """ + import questionary + + monkeypatch.setattr(onboard_commands, "_LANG", "zh") + captured: dict[str, Any] = {} + + class _FQ: + def ask(self): + return "zh" + + def _select(message, **kwargs): + captured.update(kwargs) + return _FQ() + + monkeypatch.setattr(questionary, "select", _select) + onboard_commands._pick_language() + + assert captured["default"] == "zh" + + # --------------------------------------------------------------------------- non-interactive happy path @@ -4318,6 +4363,46 @@ def fake_autocomplete(label, **kwargs): assert chosen == "openai-codex/gpt-5.6-sol" +def test_an_already_routing_current_model_stays_the_offered_default(monkeypatch: pytest.MonkeyPatch) -> None: + """A model already pointed at this provider must stay the prompt's default. + + Every other ``_pick_model`` test in this file passes ``current_model=None``, + so the branch that seeds ``default_value`` from an already-configured model + had nothing asserting it: deleting it left every one of them green, and the + prompt would have silently fallen back to the newest account model instead + of what was already set. + """ + import questionary + + from raven.providers.registry import find_by_name + + captured: dict = {} + + class _FQ: + def ask(self): + return "openai-codex/gpt-5.4" + + def fake_autocomplete(label, **kwargs): + captured.update(kwargs) + return _FQ() + + monkeypatch.setattr(questionary, "autocomplete", fake_autocomplete) + monkeypatch.setattr(onboard_commands, "_require_questionary", lambda: questionary) + + chosen = onboard_commands._pick_model( + "openai_codex", + find_by_name("openai_codex"), + current_model="openai-codex/gpt-5.4", + model_ids=["gpt-5.6-sol", "gpt-5.4"], + probe_status="valid", + user_provided_model=None, + non_interactive=False, + ) + + assert captured["default"] == "openai-codex/gpt-5.4", "the already-configured model was not offered as default" + assert chosen == "openai-codex/gpt-5.4" + + def test_a_cleared_model_prompt_says_which_one_it_fell_back_to( monkeypatch: pytest.MonkeyPatch, capsys, From 85d454458d1c1dc92ff9c7cb687b0ee8c3134473 Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 21:34:42 +0800 Subject: [PATCH 11/78] fix(tui): replay a signal swallowed during an oauth handoff suspendForHandoff defers signal exits while a device-flow login owns the terminal, but the handler dropped the signal outright: a SIGHUP landing in that window (terminal gone, delivered to the whole process group) killed the login subprocess yet left the TUI process alive with no controlling terminal -- a headless process only kill could reach. The handler now records the first deferred signal and the outermost release replays it through the normal exit path, so the intent the user's terminal expressed during the handoff is honored right after it. Co-authored-by: Claude (claude-fable-5) --- .../src/__tests__/gracefulExitDefer.test.ts | 87 +++++++++++++++++++ ui-tui/src/lib/gracefulExit.ts | 14 +++ 2 files changed, 101 insertions(+) diff --git a/ui-tui/src/__tests__/gracefulExitDefer.test.ts b/ui-tui/src/__tests__/gracefulExitDefer.test.ts index b023dfe4..844a4479 100644 --- a/ui-tui/src/__tests__/gracefulExitDefer.test.ts +++ b/ui-tui/src/__tests__/gracefulExitDefer.test.ts @@ -59,4 +59,91 @@ describe('deferSignalExit', () => { exit.mockRestore() }) + + it('replays a signal that arrived during a deferral once the deferral clears', async () => { + vi.resetModules() + const fresh = await import('../lib/gracefulExit.js') + + const cleanup = vi.fn() + const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never) + fresh.setupGracefulExit({ cleanups: [cleanup], failsafeMs: 5 }) + + const restore = fresh.deferSignalExit() + process.emit('SIGHUP') + await new Promise(resolve => setTimeout(resolve, 20)) + expect(cleanup).not.toHaveBeenCalled() + + restore() + await new Promise(resolve => setTimeout(resolve, 20)) + expect(cleanup).toHaveBeenCalled() + + exit.mockRestore() + }) + + it('only replays once the outermost deferral of a nested handoff clears', async () => { + vi.resetModules() + const fresh = await import('../lib/gracefulExit.js') + + const cleanup = vi.fn() + const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never) + fresh.setupGracefulExit({ cleanups: [cleanup], failsafeMs: 5 }) + + const outer = fresh.deferSignalExit() + const inner = fresh.deferSignalExit() + + process.emit('SIGHUP') + await new Promise(resolve => setTimeout(resolve, 20)) + expect(cleanup).not.toHaveBeenCalled() + + inner() + await new Promise(resolve => setTimeout(resolve, 20)) + expect(cleanup).not.toHaveBeenCalled() + + outer() + await new Promise(resolve => setTimeout(resolve, 20)) + expect(cleanup).toHaveBeenCalled() + + exit.mockRestore() + }) + + it('does not replay when no signal arrived during the deferral', async () => { + vi.resetModules() + const fresh = await import('../lib/gracefulExit.js') + + const cleanup = vi.fn() + const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never) + fresh.setupGracefulExit({ cleanups: [cleanup], failsafeMs: 5 }) + + const restore = fresh.deferSignalExit() + restore() + await new Promise(resolve => setTimeout(resolve, 20)) + + expect(cleanup).not.toHaveBeenCalled() + expect(exit).not.toHaveBeenCalled() + + exit.mockRestore() + }) + + it('keeps only the first signal when two arrive during the same deferral', async () => { + vi.resetModules() + const fresh = await import('../lib/gracefulExit.js') + + const onSignal = vi.fn() + const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never) + fresh.setupGracefulExit({ failsafeMs: 5, onSignal }) + + const restore = fresh.deferSignalExit() + process.emit('SIGHUP') + process.emit('SIGTERM') + await new Promise(resolve => setTimeout(resolve, 20)) + expect(onSignal).not.toHaveBeenCalled() + + restore() + await new Promise(resolve => setTimeout(resolve, 20)) + + expect(onSignal).toHaveBeenCalledTimes(1) + expect(onSignal).toHaveBeenCalledWith('SIGHUP') + + exit.mockRestore() + }) }) diff --git a/ui-tui/src/lib/gracefulExit.ts b/ui-tui/src/lib/gracefulExit.ts index 1ea2b0ce..8a4a47cc 100644 --- a/ui-tui/src/lib/gracefulExit.ts +++ b/ui-tui/src/lib/gracefulExit.ts @@ -13,6 +13,8 @@ const SIGNAL_EXIT_CODE: Record<'SIGHUP' | 'SIGINT' | 'SIGTERM', number> = { let wired = false let deferrals = 0 +let pendingSignal: keyof typeof SIGNAL_EXIT_CODE | undefined +let replaySignal: ((code: number, signal: NodeJS.Signals) => void) | undefined /** * Stop signals from exiting this process until the returned callback runs. @@ -32,6 +34,12 @@ export function deferSignalExit(): () => void { released = true deferrals = Math.max(0, deferrals - 1) + + if (deferrals === 0 && pendingSignal) { + const sig = pendingSignal + pendingSignal = undefined + replaySignal?.(SIGNAL_EXIT_CODE[sig], sig) + } } } @@ -60,9 +68,15 @@ export function setupGracefulExit({ cleanups = [], failsafeMs = 4000, onError, o void Promise.allSettled(cleanups.map(fn => Promise.resolve().then(fn))).finally(() => process.exit(code)) } + replaySignal = exit + for (const sig of ['SIGINT', 'SIGTERM', 'SIGHUP'] as const) { process.on(sig, () => { if (deferrals > 0) { + // Keep the first signal: whichever arrived first is the intent the + // process should honor once the deferral clears, later ones during + // the same handoff carry no extra information. + pendingSignal ??= sig return } From b13d33a1c5f20397bdc66b9e8115f496f5a9189f Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 21:49:25 +0800 Subject: [PATCH 12/78] fix(providers): keep fallback hops on wires that can actually serve them chat_with_retry walked the fallback chain by swapping the model string while the instance's key, base and gateway stayed frozen, so a direct provider sent another vendor's fallback model out under its own credentials -- usually a 400, and silently the wrong backend when two vendors share a model name. can_serve now skips such hops with a warning (gateways still answer for everything; unresolvable ids still fail loudly at the wire), comparing vendors through canonical_provider_name so config aliases do not read as cross-vendor. PerModelProvider stops funneling the whole chain into the primary model's endpoint and dispatches each hop to its own routed sub-provider, matching the base loop's continuation semantics. Per-hop identity rebuilding and per-hop wire overrides stay with the multi-endpoint work. Co-authored-by: Claude (claude-fable-5) --- raven/providers/base.py | 21 ++++++ raven/providers/litellm_provider.py | 23 +++++- raven/providers/per_model_provider.py | 35 ++++++++- tests/test_per_model_provider.py | 53 +++++++++++-- tests/test_provider_fallback_chain.py | 105 ++++++++++++++++++++++++++ 5 files changed, 229 insertions(+), 8 deletions(-) diff --git a/raven/providers/base.py b/raven/providers/base.py index 1d628f6c..476fb788 100644 --- a/raven/providers/base.py +++ b/raven/providers/base.py @@ -579,6 +579,14 @@ async def _chat_attempt_with_retry( return last_response # type: ignore[return-value] # loop always returns on the last attempt + def can_serve(self, model: str) -> bool: + """Whether this provider instance's credentials and wire can serve this model. + + Default True: the base class knows nothing about routing, and a wrong + guess must fail loudly at the wire rather than silently skip a hop. + """ + return True + @trace.instrument("llm.call", extract=semconv.llm_call) async def chat_with_retry( self, @@ -616,6 +624,19 @@ async def chat_with_retry( model_chain = [model, *(fallback_models or [])] response: LLMResponse | None = None for idx, current_model in enumerate(model_chain): + # A fallback hop that this instance's credentials/wire cannot serve + # (e.g. a direct provider whose fallback model resolves to another + # vendor) is skipped rather than sent -- the wrong key on the wrong + # wire either 400s outright or, worse, silently answers under a + # same-named model from the wrong vendor. Never skips the primary + # model: idx 0 is what the caller asked for. + if idx and not self.can_serve(current_model or ""): + logger.warning( + "Skipping fallback model={} - this provider instance cannot serve it (wrong vendor)", + current_model, + ) + continue + # The breakpoints in this payload were placed for whoever was asked # first. A fallback is a different model, often a different vendor, # and the field it does not read is billed rather than refused -- diff --git a/raven/providers/litellm_provider.py b/raven/providers/litellm_provider.py index ce115fe3..dd39adaf 100644 --- a/raven/providers/litellm_provider.py +++ b/raven/providers/litellm_provider.py @@ -18,7 +18,7 @@ from raven.providers.litellm_setup import import_litellm from raven.providers.prompt_cache import CACHE_CONTROL from raven.providers.reasoning import split_orphan_think -from raven.providers.registry import find_by_keywords, find_by_model, find_gateway +from raven.providers.registry import canonical_provider_name, find_by_keywords, find_by_model, find_gateway from raven.providers.wire import wire_model litellm = import_litellm() @@ -175,6 +175,27 @@ def _resolve_model(self, model: str) -> str: """The id this request is sent under. See ``providers.wire``.""" return wire_model(model, gateway=self._gateway) + def can_serve(self, model: str) -> bool: + """See ``LLMProvider.can_serve``. + + A gateway instance answers for any model -- it is the one deciding + which upstream vendor actually serves it, and its credentials are the + gateway's own, not tied to one vendor. A direct instance carries one + vendor's key on one wire: a model that resolves to a *different* + vendor's spec cannot be served here, since that would mean this + vendor's key answering for another vendor's model -- rejected outright, + or worse, silently answered wrong when two vendors happen to share a + model name. A model no spec resolves (custom endpoints, bare ids only + LiteLLM itself recognizes) is let through: it is not known to be wrong, + so it fails loudly at the wire instead of being guessed away here. + """ + if self._gateway is not None: + return True + spec = find_by_model(model) + if spec is None: + return True + return spec.name == canonical_provider_name(self._provider_name) + def _supports_cache_control(self, model: str) -> bool: """Return True when this request may carry cache_control blocks. diff --git a/raven/providers/per_model_provider.py b/raven/providers/per_model_provider.py index 6ee30276..8261e611 100644 --- a/raven/providers/per_model_provider.py +++ b/raven/providers/per_model_provider.py @@ -78,9 +78,42 @@ async def chat_with_retry( messages: list[dict[str, Any]], tools: list[dict[str, Any]] | None = None, model: str | None = None, + fallback_models: list[str] | None = None, **kwargs: Any, ) -> LLMResponse: - return await self._pick(model).chat_with_retry(messages, tools, model=model, **kwargs) + """Dispatch each hop of the fallback chain to its own routed endpoint. + + The base ``chat_with_retry`` (see ``LLMProvider``) runs the whole + ``[model, *fallback_models]`` chain through a single provider + instance, which is right when one instance can reach every hop -- + but here each hop may be a different ``knn``-routed endpoint (see + ``_endpoint_provider``). Picking a sub-provider once, up front, and + handing it the full chain would send every fallback model to the + *primary* model's endpoint. Instead, ``_pick`` runs per hop, and each + hop's own ``chat_with_retry`` is called with ``fallback_models=[]`` + so it keeps its own retry ladder without also retrying other hops' + endpoints. + + Continuation between hops mirrors ``LLMProvider.chat_with_retry``: + move to the next hop only on an error classified ``should_fallback`` + with a hop remaining; otherwise the response is returned as-is. + """ + model_chain = [model, *(fallback_models or [])] + response: LLMResponse | None = None + for idx, current_model in enumerate(model_chain): + response = await self._pick(current_model).chat_with_retry( + messages, tools, model=current_model, fallback_models=[], **kwargs + ) + if response.finish_reason != "error": + return response + + classification = response.error_classification or self.classify_error(content=response.content) + has_next = idx + 1 < len(model_chain) + if has_next and classification.should_fallback: + continue + return response + + return response # type: ignore[return-value] # chain always non-empty async def chat_stream( self, diff --git a/tests/test_per_model_provider.py b/tests/test_per_model_provider.py index d45d6153..3e890462 100644 --- a/tests/test_per_model_provider.py +++ b/tests/test_per_model_provider.py @@ -8,7 +8,7 @@ import pytest from raven.config.schema import ModelEndpoint -from raven.providers.base import GenerationSettings +from raven.providers.base import GenerationSettings, LLMResponse from raven.providers.litellm_provider import LiteLLMProvider from raven.providers.per_model_provider import PerModelProvider @@ -65,29 +65,70 @@ def test_generation_propagates_to_sub_providers(): @pytest.mark.asyncio async def test_chat_with_retry_dispatches_by_model(): p = _provider() - p._by_model["large"].chat_with_retry = AsyncMock(return_value="LARGE_RESP") - p._by_model["small"].chat_with_retry = AsyncMock(return_value="SMALL_RESP") + large_resp = LLMResponse(content="LARGE_RESP", finish_reason="stop") + small_resp = LLMResponse(content="SMALL_RESP", finish_reason="stop") + p._by_model["large"].chat_with_retry = AsyncMock(return_value=large_resp) + p._by_model["small"].chat_with_retry = AsyncMock(return_value=small_resp) out = await p.chat_with_retry(messages=[{"role": "user", "content": "hi"}], model="large") - assert out == "LARGE_RESP" + assert out.content == "LARGE_RESP" p._by_model["large"].chat_with_retry.assert_awaited_once() assert p._by_model["large"].chat_with_retry.call_args.kwargs["model"] == "large" + # fallback_models=[] so the sub-provider's own chain never re-tries a + # *different* hop's endpoint on top of its own retry ladder. + assert p._by_model["large"].chat_with_retry.call_args.kwargs["fallback_models"] == [] p._by_model["small"].chat_with_retry.assert_not_awaited() @pytest.mark.asyncio async def test_chat_with_retry_unknown_model_uses_fallback(): fb = _fallback() - fb.chat_with_retry = AsyncMock(return_value="FB_RESP") + fb.chat_with_retry = AsyncMock(return_value=LLMResponse(content="FB_RESP", finish_reason="stop")) p = PerModelProvider([ModelEndpoint(model="small", api_base="http://a/v1")], fallback=fb) out = await p.chat_with_retry(messages=[], model="other") - assert out == "FB_RESP" + assert out.content == "FB_RESP" fb.chat_with_retry.assert_awaited_once() +@pytest.mark.asyncio +async def test_chat_with_retry_dispatches_each_hop_to_its_own_endpoint(monkeypatch): + # The whole point of per-model routing: a fallback hop has its own + # endpoint (see _endpoint_provider), not the primary model's. Sending the + # entire chain to one sub-provider (the old behavior) would mean every + # fallback silently reused the primary's endpoint instead of its own. + seen: list[dict] = [] + + async def fake_acompletion(**kwargs): + seen.append(kwargs) + if kwargs["model"] == "openai/small": + raise RuntimeError("503 service unavailable") + message = MagicMock(content="ok-from-large", tool_calls=None) + return MagicMock(choices=[MagicMock(message=message, finish_reason="stop")], usage=None) + + monkeypatch.setattr("raven.providers.litellm_provider.acompletion", fake_acompletion) + + p = _provider() + # Zero the primary hop's own retry ladder so its exhaustion (a retryable + # classification) stays fast, while still exercising that it retries on + # its own endpoint before the chain moves to the next hop's. + p._by_model["small"]._CHAT_RETRY_DELAYS = (0, 0, 0) + + resp = await p.chat_with_retry( + messages=[{"role": "user", "content": "hi"}], + model="small", + fallback_models=["large"], + ) + + assert resp.content == "ok-from-large" + # The primary hop exhausts its own retry ladder (3 sleeping + 1 final) + # entirely against its own endpoint before the chain moves on. + calls = [(c["model"], c["api_base"], c["api_key"]) for c in seen] + assert calls == [("openai/small", "http://a/v1", "KA")] * 4 + [("openai/large", "http://b/v1", "KB")] + + def test_sub_providers_inherit_configured_model_overrides(): # Routed models are served by their own sub-providers, built here rather # than by make_provider -- so the user's overrides have to be pushed down diff --git a/tests/test_provider_fallback_chain.py b/tests/test_provider_fallback_chain.py index 652e0139..3cfd8499 100644 --- a/tests/test_provider_fallback_chain.py +++ b/tests/test_provider_fallback_chain.py @@ -7,13 +7,19 @@ - non-fallback fatal error (invalid request / context length) → no switch - a later model succeeding stops the chain - chain exhausted → last error surfaces +- a fallback hop this provider instance cannot serve (wrong vendor + credentials) is skipped rather than sent, unless the instance is a gateway """ from __future__ import annotations +import io + import pytest +from loguru import logger as _logger from raven.providers.base import LLMProvider, LLMResponse +from raven.providers.litellm_provider import LiteLLMProvider class _ScriptedProvider(LLMProvider): @@ -215,6 +221,105 @@ async def test_chat_timeout_is_retried_then_succeeds(): assert provider.calls == 3 # two timeouts retried, third succeeds +@pytest.mark.asyncio +async def test_direct_provider_skips_fallback_hop_resolved_to_another_vendor(monkeypatch): + # A direct (non-gateway) instance carries one vendor's credentials. A + # fallback hop that ``find_by_model`` resolves to a *different* vendor's + # spec must not be sent on this wire -- can_serve() should skip it, and + # the chain then exhausts on the primary's own error since no other hop + # remains. + provider = LiteLLMProvider( + api_key="sk-ant-test", default_model="anthropic/claude-opus-4-5", provider_name="anthropic" + ) + assert provider._gateway is None + + calls: list[str | None] = [] + + async def fake_chat(messages, tools=None, model=None, **kwargs): + calls.append(model) + return LLMResponse(content="model not found", finish_reason="error") + + monkeypatch.setattr(provider, "chat", fake_chat) + + captured = io.StringIO() + sink_id = _logger.add(captured, level="WARNING") + try: + resp = await provider.chat_with_retry( + messages=[], + model="anthropic/claude-opus-4-5", + fallback_models=["openai/gpt-4o"], + ) + finally: + _logger.remove(sink_id) + + assert resp.finish_reason == "error" + assert resp.content == "model not found" + # The openai hop is never dispatched -- can_serve() vetoed it. + assert calls == ["anthropic/claude-opus-4-5"] + assert "openai/gpt-4o" in captured.getvalue() + + +@pytest.mark.asyncio +async def test_gateway_provider_does_not_skip_cross_vendor_fallback_hop(monkeypatch): + # A gateway instance (OpenRouter, AiHubMix, ...) routes any model on the + # caller's behalf, so can_serve() must return True unconditionally -- + # unlike a direct instance, it must not skip a fallback hop just because + # that hop resolves to a different upstream vendor's spec. + provider = LiteLLMProvider( + api_key="sk-or-test", default_model="openrouter/anthropic/claude-opus-4-5", provider_name="openrouter" + ) + assert provider._gateway is not None + + calls: list[str | None] = [] + + async def fake_chat(messages, tools=None, model=None, **kwargs): + calls.append(model) + if model == "openrouter/anthropic/claude-opus-4-5": + return LLMResponse(content="model not found", finish_reason="error") + return LLMResponse(content="ok", finish_reason="stop") + + monkeypatch.setattr(provider, "chat", fake_chat) + + resp = await provider.chat_with_retry( + messages=[], + model="openrouter/anthropic/claude-opus-4-5", + fallback_models=["openai/gpt-4o"], + ) + + assert resp.content == "ok" + assert calls == ["openrouter/anthropic/claude-opus-4-5", "openai/gpt-4o"] + + +@pytest.mark.asyncio +async def test_chain_of_unserviceable_hops_returns_primary_error(monkeypatch): + # When every fallback hop is unserviceable (all resolve to a different + # vendor than this direct instance's own), the chain skips all of them + # and the caller gets back the primary model's own last error -- the same + # outcome as an ordinary exhausted chain, not a None or a crash. + provider = LiteLLMProvider( + api_key="sk-ant-test", default_model="anthropic/claude-opus-4-5", provider_name="anthropic" + ) + assert provider._gateway is None + + calls: list[str | None] = [] + + async def fake_chat(messages, tools=None, model=None, **kwargs): + calls.append(model) + return LLMResponse(content="model not found", finish_reason="error") + + monkeypatch.setattr(provider, "chat", fake_chat) + + resp = await provider.chat_with_retry( + messages=[], + model="anthropic/claude-opus-4-5", + fallback_models=["openai/gpt-4o", "gemini/gemini-2.5-flash"], + ) + + assert resp.finish_reason == "error" + assert resp.content == "model not found" + assert calls == ["anthropic/claude-opus-4-5"] + + @pytest.mark.asyncio async def test_should_fallback_classification(): # Structured classifier (string path): transient + capacity/availability From 0d9c78434d40c6d951ca87654372840e2a2e2f76 Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 21:54:54 +0800 Subject: [PATCH 13/78] chore(providers): let ruff format pick the quote style in the refusal table Co-authored-by: Claude (claude-fable-5) --- raven/providers/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raven/providers/auth.py b/raven/providers/auth.py index 14e84dcd..77462ba2 100644 --- a/raven/providers/auth.py +++ b/raven/providers/auth.py @@ -280,7 +280,7 @@ def credential_status( "ChatGPT is reached through its own OAuth device flow -- LiteLLM's " "chatgpt transformation ignores any api_key and authenticates through a " "stored browser session instead. Raven already has this path: run " - "`raven provider login openai-codex` (or pick \"OpenAI Codex (OAuth)\" " + '`raven provider login openai-codex` (or pick "OpenAI Codex (OAuth)" ' "from this menu)." ), "bedrock": ( From d614355a8b1a5725171b4a8af75566655055653d Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 22:06:51 +0800 Subject: [PATCH 14/78] fix(providers): stop masking chat errors as assistant text in stream mode A provider without real streaming (azure, codex) replays its chat() result through the default chat_stream fallback as one terminal StreamDelta -- which had nowhere to put finish_reason or error_classification, and the stream collation then hardcoded the finish_reason to stop/tool_calls. Net effect on the default streaming path: an upstream error was rendered to the user as a normal assistant reply, with no retry, no fallback, and the turn recorded as success. The terminal delta now carries both fields, and _llm_call_stream diverts an error delta away from token rendering into the same error LLMResponse shape the non-streaming path already returns, so the existing recovery and terminal-error handling apply unchanged. Co-authored-by: Claude (claude-fable-5) --- raven/agent/loop/main.py | 25 ++++++++++++++++- raven/providers/base.py | 8 ++++++ tests/test_agent_loop_stream.py | 37 +++++++++++++++++++++++++- tests/test_provider_stream_fallback.py | 22 ++++++++++++++- 4 files changed, 89 insertions(+), 3 deletions(-) diff --git a/raven/agent/loop/main.py b/raven/agent/loop/main.py index f99b5a31..7ebb6e39 100644 --- a/raven/agent/loop/main.py +++ b/raven/agent/loop/main.py @@ -43,7 +43,7 @@ from raven.agent.tools.web import WebFetchTool, WebSearchTool from raven.memory_engine.base import TokenBudget from raven.memory_engine.consolidate.consolidator import MemoryConsolidator, MemoryStore -from raven.providers.base import LLMProvider, LLMResponse, ToolCallRequest +from raven.providers.base import ErrorClassification, LLMProvider, LLMResponse, ToolCallRequest from raven.providers.capabilities import image_placeholder_text, supports_image_tool_result, vision_verdict from raven.providers.rates import effective_context_window, resolve_context_window from raven.providers.reasoning import split_orphan_think @@ -1560,6 +1560,9 @@ async def _llm_call_stream( reasoning_buf: list[str] = [] tool_call_slots: list[dict[str, Any]] = [] final_usage: dict[str, Any] | None = None + had_error = False + error_content: str | None = None + error_classification: ErrorClassification | None = None # aclosing() guarantees the async generator (and its underlying stream) # is closed when a TimeoutError from the per-chunk idle cap unwinds the @@ -1569,6 +1572,18 @@ async def _llm_call_stream( try: async with aclosing(self.provider.chat_stream(messages=messages, tools=tools, model=model)) as stream: async for delta in stream: + if delta.finish_reason == "error": + # A non-streaming provider's chat() error, replayed + # through the fallback as its single terminal delta. + # Its content is the error text, not a token to render + # or accumulate -- surface it via error_classification + # instead of the normal success collation below. + had_error = True + error_content = delta.content + error_classification = delta.error_classification + if delta.usage is not None: + final_usage = delta.usage + continue reasoning_delta = getattr(delta, "reasoning_content", None) if reasoning_delta: reasoning_buf.append(reasoning_delta) @@ -1592,6 +1607,14 @@ async def _llm_call_stream( error_classification=self.provider.classify_error(TimeoutError()), ) + if had_error: + return LLMResponse( + content=error_content, + finish_reason="error", + error_classification=error_classification, + usage=final_usage or {}, + ) + tool_calls = _finalize_tool_calls(tool_call_slots) finish_reason = "tool_calls" if tool_calls else "stop" diff --git a/raven/providers/base.py b/raven/providers/base.py index 476fb788..4436ebb1 100644 --- a/raven/providers/base.py +++ b/raven/providers/base.py @@ -124,12 +124,18 @@ class StreamDelta: Consumers (AgentLoop on_token_delta path, TUI SubscriptionEmitter) read `.content` for incremental token text; `tool_call_delta` / `usage` are optional carriers for in-stream tool deltas and final usage snapshots. + + `finish_reason` / `error_classification` are only ever set on the + terminal delta of a stream (mirroring `LLMResponse`); mid-stream deltas + leave both as ``None``. """ content: str | None tool_call_delta: dict[str, Any] | None = None usage: dict[str, Any] | None = None reasoning_content: str | None = None # Kimi, DeepSeek-R1, qwen, o-series thinking stream + finish_reason: str | None = None + error_classification: ErrorClassification | None = None @dataclass(frozen=True) @@ -300,6 +306,8 @@ async def chat_stream( tool_call_delta=tool_call_delta, usage=response.usage or None, reasoning_content=response.reasoning_content, + finish_reason=response.finish_reason, + error_classification=response.error_classification, ) @staticmethod diff --git a/tests/test_agent_loop_stream.py b/tests/test_agent_loop_stream.py index fc30c102..f5d5f25e 100644 --- a/tests/test_agent_loop_stream.py +++ b/tests/test_agent_loop_stream.py @@ -12,7 +12,7 @@ from typing import Any from raven.agent.loop import AgentLoop -from raven.providers.base import LLMProvider, LLMResponse, StreamDelta +from raven.providers.base import ErrorClassification, LLMProvider, LLMResponse, StreamDelta class _FakeProvider: @@ -245,6 +245,41 @@ async def on_delta(text: str) -> None: assert seen == ["partial"] +async def test_llm_call_stream_error_delta_is_not_rendered_as_a_token() -> None: + """A non-streaming provider's chat() error, replayed through the base + fallback as a single terminal delta with finish_reason='error', must not + be treated as ordinary streamed content: on_token_delta must not fire for + it, and the final response must surface finish_reason + classification + instead of a fabricated 'stop'/'tool_calls'.""" + classification = ErrorClassification(category="http_4xx", should_fallback=True) + chunks = [ + StreamDelta( + content="Azure OpenAI API Error 404: deployment not found", + finish_reason="error", + error_classification=classification, + ), + ] + provider = _FakeProvider(chunks) + call = _bind_helper(provider) + + seen: list[str] = [] + + async def on_delta(text: str) -> None: + seen.append(text) + + response = await call( + messages=[{"role": "user", "content": "hi"}], + tools=None, + model="m", + on_token_delta=on_delta, + ) + + assert seen == [] + assert response.content == "Azure OpenAI API Error 404: deployment not found" + assert response.finish_reason == "error" + assert response.error_classification is classification + + async def test_llm_call_stream_empty_stream_yields_empty_content() -> None: """Provider yields zero chunks → response.content == '' + finish_reason='stop'.""" provider = _FakeProvider([]) diff --git a/tests/test_provider_stream_fallback.py b/tests/test_provider_stream_fallback.py index 3428acf5..276ce950 100644 --- a/tests/test_provider_stream_fallback.py +++ b/tests/test_provider_stream_fallback.py @@ -13,7 +13,7 @@ import pytest -from raven.providers.base import GenerationSettings, LLMProvider, LLMResponse, ToolCallRequest +from raven.providers.base import ErrorClassification, GenerationSettings, LLMProvider, LLMResponse, ToolCallRequest class _ChatOnlyProvider(LLMProvider): @@ -49,6 +49,26 @@ async def test_fallback_yields_single_terminal_delta() -> None: assert deltas[0].tool_call_delta is None +async def test_fallback_propagates_error_finish_reason_and_classification() -> None: + """A chat() error response (e.g. Azure non-200) must not be silently + stripped down to plain content when replayed through the terminal delta -- + the caller needs finish_reason + error_classification to detect it.""" + classification = ErrorClassification(category="http_4xx", should_fallback=True) + provider = _ChatOnlyProvider( + LLMResponse( + content="Azure OpenAI API Error 404: deployment not found", + finish_reason="error", + error_classification=classification, + ) + ) + deltas = [d async for d in provider.chat_stream(messages=[{"role": "user", "content": "hi"}])] + + assert len(deltas) == 1 + assert deltas[0].content == "Azure OpenAI API Error 404: deployment not found" + assert deltas[0].finish_reason == "error" + assert deltas[0].error_classification is classification + + async def test_fallback_encodes_tool_calls_for_reconstruction() -> None: provider = _ChatOnlyProvider( LLMResponse( From 30e7c9ec2a971bead74318a1baaf3631ecdb0078 Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 22:09:10 +0800 Subject: [PATCH 15/78] fix(providers): classify a rendered 404 body as model_unavailable The 429 and 5xx buckets each carry their literal status as a substring marker for the degraded string path, but the 404 bucket only had the wordier phrasings -- so a provider that renders its non-200 body into a plain string (azure's path) with a route-level "Resource not found" message fell through to unknown and lost the fallback. Co-authored-by: Claude (claude-fable-5) --- raven/providers/base.py | 6 ++++++ tests/test_error_classification.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/raven/providers/base.py b/raven/providers/base.py index 4436ebb1..f8113bc3 100644 --- a/raven/providers/base.py +++ b/raven/providers/base.py @@ -455,6 +455,11 @@ def has(*needles: str) -> bool: return ErrorClassification("billing", should_fallback=True) # Model unavailable / not found → no point retrying it; try another model. + # "404" as a substring mirrors the 429/5xx buckets above: a provider + # that embeds the status into a rendered string (azure's non-200 path) + # reaches here with no exception to read a status code from, and a + # route-level body like "Resource not found" names none of the wordier + # markers. if ( status == 404 or "notfounderror" in names @@ -464,6 +469,7 @@ def has(*needles: str) -> bool: "no endpoints", "not available", "unavailable", + "404", ) ): return ErrorClassification("model_unavailable", should_fallback=True) diff --git a/tests/test_error_classification.py b/tests/test_error_classification.py index 07ea1ad1..05cb58dd 100644 --- a/tests/test_error_classification.py +++ b/tests/test_error_classification.py @@ -97,6 +97,9 @@ def test_classify_follows_cause_chain(): ("connection reset by peer", "network"), ("insufficient credit / billing", "billing"), ("model not found", "model_unavailable"), + # A rendered azure non-200 body: no exception, no status attribute, + # and a route-level 404 text that names none of the wordier markers. + ("Azure OpenAI API Error 404: Resource not found", "model_unavailable"), ("This model's maximum context length is 8192 tokens", "context_overflow"), ("401 unauthorized: invalid api key", "auth"), ("400 invalid request: bad schema", "invalid_request"), From 88bdba07eee1bffef63254599173beb67630c662 Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 23:45:30 +0800 Subject: [PATCH 16/78] fix(providers): only veto fallback hops both identities are certain about can_serve treated "my provider_name resolves to nothing" as a mismatch, so an instance constructed without a provider_name -- the proactive planner and evolver both build one that way -- skipped every resolvable fallback hop, and an OAuth identity like github_copilot, whose one grant serves several upstream vendors, refused models it actually serves. The veto now requires both sides to be known, non-OAuth and different; every unresolved identity falls back to failing loudly at the wire, which is the base class's stated default. Co-authored-by: Claude (claude-fable-5) --- raven/providers/litellm_provider.py | 45 ++++++++++++++++++++------- tests/test_provider_fallback_chain.py | 31 ++++++++++++++++++ 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/raven/providers/litellm_provider.py b/raven/providers/litellm_provider.py index dd39adaf..44fb231f 100644 --- a/raven/providers/litellm_provider.py +++ b/raven/providers/litellm_provider.py @@ -18,7 +18,13 @@ from raven.providers.litellm_setup import import_litellm from raven.providers.prompt_cache import CACHE_CONTROL from raven.providers.reasoning import split_orphan_think -from raven.providers.registry import canonical_provider_name, find_by_keywords, find_by_model, find_gateway +from raven.providers.registry import ( + canonical_provider_name, + find_by_keywords, + find_by_model, + find_by_name, + find_gateway, +) from raven.providers.wire import wire_model litellm = import_litellm() @@ -180,21 +186,36 @@ def can_serve(self, model: str) -> bool: A gateway instance answers for any model -- it is the one deciding which upstream vendor actually serves it, and its credentials are the - gateway's own, not tied to one vendor. A direct instance carries one - vendor's key on one wire: a model that resolves to a *different* - vendor's spec cannot be served here, since that would mean this - vendor's key answering for another vendor's model -- rejected outright, - or worse, silently answered wrong when two vendors happen to share a - model name. A model no spec resolves (custom endpoints, bare ids only - LiteLLM itself recognizes) is let through: it is not known to be wrong, - so it fails loudly at the wire instead of being guessed away here. + gateway's own, not tied to one vendor. + + For a direct instance, this only vetoes the one case both sides are + certain about: this instance's own provider_name resolves to a known, + non-OAuth spec, the model resolves to a *different* known spec, and + the two disagree -- that is one vendor's key answering for another + vendor's model, rejected outright. Every other case is let through + rather than guessed away here: + - this instance's own identity does not resolve to a spec (empty + provider_name, "auto", or a custom passthrough name LiteLLM + recognizes natively but Raven has no ProviderSpec for, e.g. + nebius/fireworks/together) -- there is nothing to compare against; + - the resolved spec is OAuth-based (e.g. github_copilot): one OAuth + grant can serve several upstream vendors, so a spec mismatch there + says nothing about whether this instance can serve the model; + - the model resolves to no spec at all (custom endpoints, bare ids + only LiteLLM itself recognizes). + In all of those, the model is not known to be wrong for this + instance, so it fails loudly at the wire instead of being guessed + away here. """ if self._gateway is not None: return True - spec = find_by_model(model) - if spec is None: + mine = find_by_name(canonical_provider_name(self._provider_name)) + if mine is None or mine.is_oauth: + return True + theirs = find_by_model(model) + if theirs is None: return True - return spec.name == canonical_provider_name(self._provider_name) + return theirs.name == mine.name def _supports_cache_control(self, model: str) -> bool: """Return True when this request may carry cache_control blocks. diff --git a/tests/test_provider_fallback_chain.py b/tests/test_provider_fallback_chain.py index 3cfd8499..f4c073be 100644 --- a/tests/test_provider_fallback_chain.py +++ b/tests/test_provider_fallback_chain.py @@ -320,6 +320,37 @@ async def fake_chat(messages, tools=None, model=None, **kwargs): assert calls == ["anthropic/claude-opus-4-5"] +def test_empty_provider_name_does_not_skip_resolvable_fallback(): + # provider_name="" (the real construction site default: neither + # _proactive_stack.py nor evolver/launch/models.py passes provider_name) + # resolves to no spec at all, so this instance's own identity is unknown + # -- can_serve must let every resolvable model through rather than + # comparing an unknown identity against a known one and rejecting. + provider = LiteLLMProvider(api_key="test-key", default_model="anthropic/claude-opus-4-5", provider_name="") + assert provider._gateway is None + assert provider.can_serve("anthropic/claude-opus-4-5") is True + assert provider.can_serve("openai/gpt-4o") is True + + +def test_oauth_identity_does_not_skip_cross_vendor_model(): + # github_copilot is a single OAuth grant that can serve several upstream + # vendors (OpenAI, Anthropic, Google), so a spec mismatch says nothing + # about whether this instance can serve the model. + provider = LiteLLMProvider(default_model="github_copilot/gpt-4o", provider_name="github_copilot") + assert provider._gateway is None + assert provider.can_serve("anthropic/claude-opus-4-5") is True + + +def test_custom_passthrough_identity_does_not_skip_resolvable_fallback(): + # "fireworks" has no ProviderSpec -- it is a direct-connect vendor LiteLLM + # supports natively under its own routing prefix. This instance's own + # identity does not resolve to a spec, so it cannot be compared against + # the fallback model's resolved spec. + provider = LiteLLMProvider(api_key="test-key", default_model="fireworks/some-model", provider_name="fireworks") + assert provider._gateway is None + assert provider.can_serve("anthropic/claude-opus-4-5") is True + + @pytest.mark.asyncio async def test_should_fallback_classification(): # Structured classifier (string path): transient + capacity/availability From cfdeb19d5e9a737423f1372a3c013090074e90d7 Mon Sep 17 00:00:00 2001 From: KT Date: Sun, 9 Aug 2026 23:47:17 +0800 Subject: [PATCH 17/78] fix(providers): let explicit model_overrides win over shipped extra_body The merge gave the provider's built-in extra_body priority on collisions, justified by "routing pins that must reach the wire" -- but the whole _WIRE_OVERRIDES table is one shipped default workaround (disabling OpenRouter's qwen reasoning mode), not a routing pin, and the same change set documented model_overrides as the channel that overrides shipped defaults. A collision is the user deliberately reversing one, so the user's value wins now; the conflict test mirrors the one real collision instead of an invented provider pin. Co-authored-by: Claude (claude-fable-5) --- raven/providers/litellm_provider.py | 13 ++++++++----- tests/test_litellm_provider_model_overrides.py | 16 ++++++++++------ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/raven/providers/litellm_provider.py b/raven/providers/litellm_provider.py index 44fb231f..713a0169 100644 --- a/raven/providers/litellm_provider.py +++ b/raven/providers/litellm_provider.py @@ -67,18 +67,21 @@ def _short_tool_id() -> str: def _merge_extra_body(kwargs: dict[str, Any], wire_extra_body: dict[str, Any]) -> None: - """Merge the provider's wire-routing extra_body into kwargs instead of overwriting it. + """Merge the provider's built-in extra_body into kwargs instead of overwriting it. A model_overrides entry (see _apply_model_overrides) may have already placed a user extra_body dict in kwargs -- for example Qwen3's extra_body.chat_template_kwargs.enable_thinking. Assigning wire_extra_body - over it would silently drop those keys. On a key collision, wire_extra_body - wins: it carries routing pins (e.g. OpenRouter provider order) that must - reach the wire intact. + over it would silently drop those keys. On a key collision, the user's + value wins: everything wire_extra_body carries is a shipped default + workaround (see capabilities._WIRE_OVERRIDES -- disabling OpenRouter's + qwen reasoning mode is the whole table today), and model_overrides is + documented as the channel that overrides shipped defaults, so a collision + is the user deliberately reversing one. """ existing = kwargs.get("extra_body") if isinstance(existing, dict): - kwargs["extra_body"] = {**existing, **wire_extra_body} + kwargs["extra_body"] = {**wire_extra_body, **existing} else: kwargs["extra_body"] = wire_extra_body diff --git a/tests/test_litellm_provider_model_overrides.py b/tests/test_litellm_provider_model_overrides.py index cb1d50c2..bb693ede 100644 --- a/tests/test_litellm_provider_model_overrides.py +++ b/tests/test_litellm_provider_model_overrides.py @@ -171,22 +171,26 @@ async def test_model_override_extra_body_merges_with_wire_routing(monkeypatch: p @pytest.mark.asyncio -async def test_model_override_extra_body_wire_key_wins_on_conflict(monkeypatch: pytest.MonkeyPatch) -> None: - # On a colliding key, the provider's own wire-routing extra_body must win -- - # routing correctness over a user override that would misroute the request. +async def test_model_override_extra_body_user_key_wins_on_conflict(monkeypatch: pytest.MonkeyPatch) -> None: + # On a colliding key, the user's model_overrides value must win: everything + # the provider ships in extra_body is a default workaround (the whole + # _WIRE_OVERRIDES table is one qwen reasoning switch), and model_overrides + # is documented as the channel that overrides shipped defaults. Mirrors the + # real collision: the shipped {"reasoning": {"enabled": False}} against a + # user who wants reasoning back on. seen = _capture(monkeypatch) p = LiteLLMProvider( api_key="test-key", default_model="openrouter/qwen/qwen3", provider_name="openrouter", - extra_body={"provider": {"order": ["Anthropic"]}}, - model_overrides={"qwen3": {"extra_body": {"provider": {"order": ["Alibaba"]}}}}, + extra_body={"reasoning": {"enabled": False}}, + model_overrides={"qwen3": {"extra_body": {"reasoning": {"enabled": True}}}}, ) p.generation = GenerationSettings(temperature=0.1) await p.chat(messages=[{"role": "user", "content": "hi"}]) - assert seen[0]["extra_body"] == {"provider": {"order": ["Anthropic"]}} + assert seen[0]["extra_body"] == {"reasoning": {"enabled": True}} @pytest.mark.asyncio From 24cc9e7ac2a978990fc46fd314bd2022db6dab56 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 00:15:35 +0800 Subject: [PATCH 18/78] feat(*): declare provider endpoints and resolve every spelling in one place First stage of multi-endpoint support: ProviderConfig gains an endpoints list (label + key + base + headers per entry), and providers/endpoints.py resolves the three spellings a section can carry -- explicit endpoints, Gemini's api_key_list, the flat fields -- into one uniform list with strict precedence and no merging, so a stale flat key cannot outlive the endpoint meant to replace it. Reading the shapes independently is how the Gemini list came to be declared and never used. Rotation and failover read this list next; make_provider will refuse the field on OAuth/azure/codex providers when it is wired up. Co-authored-by: Claude (claude-fable-5) --- raven/config/schema.py | 25 ++++++++ raven/providers/endpoints.py | 81 ++++++++++++++++++++++++ tests/test_config_schema.py | 54 ++++++++++++++-- tests/test_provider_endpoints.py | 105 +++++++++++++++++++++++++++++++ 4 files changed, 260 insertions(+), 5 deletions(-) create mode 100644 raven/providers/endpoints.py create mode 100644 tests/test_provider_endpoints.py diff --git a/raven/config/schema.py b/raven/config/schema.py index 9bffb253..e1c0f208 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -340,6 +340,20 @@ class ModelOverlay(Base): description: str = "" +class ProviderEndpoint(Base): + """One named URL/key group under a provider section. + + ``label`` is not decoration: it is the idempotency key a later stage + (rotation, failover, per-endpoint health) uses to address one entry across + edits, so two endpoints in the same list must not share one. + """ + + label: str + api_key: str = "" + api_base: str | None = None + extra_headers: dict[str, str] | None = None + + class ProviderConfig(Base): """LLM provider configuration.""" @@ -347,6 +361,17 @@ class ProviderConfig(Base): api_base: str | None = None extra_headers: dict[str, str] | None = None # Custom headers (e.g. APP-Code for AiHubMix) models: list[str] = Field(default_factory=list) # User-curated model names for the picker + # Several full url/key/header groups under one provider section, for a + # vendor reachable by more than one account or region. Meaningful only for + # a plain API-key provider reached through the litellm client -- a section + # whose auth is OAuth, or that needs more than a key and an address (Azure + # OpenAI, Codex), gets this rejected at `make_provider` construction time + # (wired in a later stage; this field exists regardless). Set and non-empty, + # it replaces the flat `api_key`/`api_base` outright rather than merging + # with them -- see `raven.providers.endpoints.provider_endpoints` for the + # one place that resolves which of the two shapes (or Gemini's + # `api_key_list`) is in effect. + endpoints: list[ProviderEndpoint] = Field(default_factory=list) # Keyed by model id, in any spelling: what the user knows about a model that # the catalogues do not. Deliberately additive rather than a change to # `models` -- that list already lets a model be added, and what was missing diff --git a/raven/providers/endpoints.py b/raven/providers/endpoints.py new file mode 100644 index 00000000..d557330e --- /dev/null +++ b/raven/providers/endpoints.py @@ -0,0 +1,81 @@ +"""Where a provider's connection material comes from, however the section spells it. + +A provider config has carried the same three things -- key, address, headers -- +under three different shapes: the flat ``api_key``/``api_base`` every section +has; Gemini's ``api_key_list`` (several keys, one section, sharing the flat +address); and now ``endpoints`` (several full label/key/base/headers groups, +the material S2's rotation and failover will read). Reading any one of them +independently is how the Gemini list came to be declared and never used -- +``GeminiProviderConfig.effective_api_key`` reads only the first key, so listing +several kept exactly one of them alive. + +``provider_endpoints`` is the one place that resolves the three shapes into a +uniform list, so "every endpoint this section offers" is asked once rather +than re-derived at each call site with its own idea of the precedence. + +The three do not mix. ``endpoints`` set means the flat fields and +``api_key_list`` are both ignored outright, not merged with the list -- a +partial merge is how a stale flat key would outlive the endpoint meant to +replace it. ``api_key_list`` without ``endpoints`` still shares the flat +``api_base``/``extra_headers``: those were never plural, so there is nothing +to choose between for them. + +An unconfigured section (no endpoints, no list, no flat key) resolves to one +endpoint holding the empty flat values rather than an empty list. That is what +every existing caller already got from reading the flat fields directly before +this module existed, so returning nothing here would just move the "now what" +onto each of them instead of answering it once. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from raven.config.schema import ProviderConfig + + +@dataclass(frozen=True) +class ResolvedEndpoint: + """One url/key/header group, whichever shape it was read from.""" + + label: str + api_key: str + api_base: str | None + extra_headers: dict[str, str] | None + + +def provider_endpoints(section: "ProviderConfig") -> list[ResolvedEndpoint]: + """Every endpoint ``section`` offers, in the shape it was declared.""" + if section.endpoints: + return [ + ResolvedEndpoint( + label=endpoint.label, + api_key=endpoint.api_key, + api_base=endpoint.api_base, + extra_headers=endpoint.extra_headers, + ) + for endpoint in section.endpoints + ] + + key_list = getattr(section, "api_key_list", None) + if key_list: + return [ + ResolvedEndpoint( + label=f"key-{i}", + api_key=key, + api_base=section.api_base, + extra_headers=section.extra_headers, + ) + for i, key in enumerate(key_list, start=1) + ] + + return [ + ResolvedEndpoint( + label="default", + api_key=section.api_key, + api_base=section.api_base, + extra_headers=section.extra_headers, + ) + ] diff --git a/tests/test_config_schema.py b/tests/test_config_schema.py index 458bfe09..7b04d666 100644 --- a/tests/test_config_schema.py +++ b/tests/test_config_schema.py @@ -1,13 +1,20 @@ -"""Tests for ``raven.config.schema.AgentDefaults.context_window_tokens``. +"""Tests for ``raven.config.schema.AgentDefaults.context_window_tokens`` and +``ProviderConfig.endpoints``. -None (or 0) means "figure it out" against the model's real window; a positive -value pins it. See ``raven.providers.rates.effective_context_window`` for the -ladder that reads this field. +The context-window tests: None (or 0) means "figure it out" against the +model's real window; a positive value pins it. See +``raven.providers.rates.effective_context_window`` for the ladder that reads +this field. + +The endpoints tests: ``ProviderEndpoint`` is the S1 data shape for a provider +section holding several url/key/header groups. See +``raven.providers.endpoints.provider_endpoints`` for the read point that +resolves it against the older flat/``api_key_list`` shapes. """ from __future__ import annotations -from raven.config.schema import AgentDefaults +from raven.config.schema import AgentDefaults, ProviderConfig, ProviderEndpoint def test_context_window_tokens_defaults_to_none() -> None: @@ -23,3 +30,40 @@ def test_context_window_tokens_camel_alias_round_trips() -> None: defaults = AgentDefaults.model_validate({"contextWindowTokens": 200_000}) assert defaults.context_window_tokens == 200_000 assert defaults.model_dump(by_alias=True)["contextWindowTokens"] == 200_000 + + +def test_provider_endpoints_defaults_to_empty_list() -> None: + assert ProviderConfig().endpoints == [] + + +def test_provider_endpoint_round_trips_with_camel_alias() -> None: + endpoint = ProviderEndpoint.model_validate( + {"label": "us-east", "apiKey": "sk-1", "apiBase": "https://a.example", "extraHeaders": {"X-Region": "us"}} + ) + assert endpoint.label == "us-east" + assert endpoint.api_key == "sk-1" + assert endpoint.api_base == "https://a.example" + assert endpoint.extra_headers == {"X-Region": "us"} + + dumped = endpoint.model_dump(by_alias=True) + assert dumped["apiKey"] == "sk-1" + assert dumped["apiBase"] == "https://a.example" + assert dumped["extraHeaders"] == {"X-Region": "us"} + + +def test_provider_endpoint_defaults() -> None: + endpoint = ProviderEndpoint(label="only") + assert endpoint.api_key == "" + assert endpoint.api_base is None + assert endpoint.extra_headers is None + + +def test_provider_config_endpoints_round_trip_with_camel_alias() -> None: + section = ProviderConfig.model_validate( + {"endpoints": [{"label": "primary", "apiKey": "sk-1"}, {"label": "backup", "apiKey": "sk-2"}]} + ) + assert [e.label for e in section.endpoints] == ["primary", "backup"] + + dumped = section.model_dump(by_alias=True) + assert dumped["endpoints"][0]["apiKey"] == "sk-1" + assert dumped["endpoints"][1]["label"] == "backup" diff --git a/tests/test_provider_endpoints.py b/tests/test_provider_endpoints.py new file mode 100644 index 00000000..9e3510d0 --- /dev/null +++ b/tests/test_provider_endpoints.py @@ -0,0 +1,105 @@ +"""Tests for raven.providers.endpoints -- the one read point over a provider +section's three ways of holding url/key/header material. + +The three shapes (``endpoints``, Gemini's ``api_key_list``, the flat fields) +do not mix: whichever is most specific wins outright, not merged with the +others. These assert that precedence and the empty-section fallback. +""" + +from __future__ import annotations + +from raven.config.schema import GeminiProviderConfig, ProviderConfig, ProviderEndpoint +from raven.providers.endpoints import ResolvedEndpoint, provider_endpoints + + +def test_flat_fields_synthesize_a_single_default_endpoint() -> None: + section = ProviderConfig(api_key="sk-flat", api_base="https://flat.example", extra_headers={"X-A": "1"}) + + assert provider_endpoints(section) == [ + ResolvedEndpoint( + label="default", api_key="sk-flat", api_base="https://flat.example", extra_headers={"X-A": "1"} + ) + ] + + +def test_empty_section_resolves_to_one_empty_endpoint_not_an_empty_list() -> None: + assert provider_endpoints(ProviderConfig()) == [ + ResolvedEndpoint(label="default", api_key="", api_base=None, extra_headers=None) + ] + + +def test_endpoints_list_is_used_verbatim() -> None: + section = ProviderConfig( + endpoints=[ + ProviderEndpoint(label="primary", api_key="sk-1", api_base="https://a.example"), + ProviderEndpoint(label="backup", api_key="sk-2", api_base="https://b.example", extra_headers={"X-B": "2"}), + ] + ) + + assert provider_endpoints(section) == [ + ResolvedEndpoint(label="primary", api_key="sk-1", api_base="https://a.example", extra_headers=None), + ResolvedEndpoint(label="backup", api_key="sk-2", api_base="https://b.example", extra_headers={"X-B": "2"}), + ] + + +def test_endpoints_list_takes_priority_over_flat_fields_not_merged() -> None: + section = ProviderConfig( + api_key="sk-flat", + api_base="https://flat.example", + extra_headers={"X-Flat": "1"}, + endpoints=[ProviderEndpoint(label="only")], + ) + + resolved = provider_endpoints(section) + + assert resolved == [ResolvedEndpoint(label="only", api_key="", api_base=None, extra_headers=None)] + + +def test_gemini_api_key_list_yields_one_endpoint_per_key() -> None: + section = GeminiProviderConfig(api_key_list=["k1", "k2", "k3"], api_base="https://gemini.example") + + assert provider_endpoints(section) == [ + ResolvedEndpoint(label="key-1", api_key="k1", api_base="https://gemini.example", extra_headers=None), + ResolvedEndpoint(label="key-2", api_key="k2", api_base="https://gemini.example", extra_headers=None), + ResolvedEndpoint(label="key-3", api_key="k3", api_base="https://gemini.example", extra_headers=None), + ] + + +def test_gemini_api_key_list_shares_the_flat_api_base_and_headers() -> None: + section = GeminiProviderConfig( + api_key_list=["k1", "k2"], api_base="https://gemini.example", extra_headers={"X-G": "1"} + ) + + resolved = provider_endpoints(section) + + assert all(e.api_base == "https://gemini.example" for e in resolved) + assert all(e.extra_headers == {"X-G": "1"} for e in resolved) + + +def test_gemini_api_key_list_takes_priority_over_flat_api_key_without_duplication() -> None: + section = GeminiProviderConfig(api_key="sk-flat", api_key_list=["k1", "k2"]) + + resolved = provider_endpoints(section) + + assert [e.api_key for e in resolved] == ["k1", "k2"] + assert "sk-flat" not in [e.api_key for e in resolved] + + +def test_gemini_without_api_key_list_falls_back_to_flat_fields() -> None: + section = GeminiProviderConfig(api_key="sk-flat", api_base="https://gemini.example") + + assert provider_endpoints(section) == [ + ResolvedEndpoint(label="default", api_key="sk-flat", api_base="https://gemini.example", extra_headers=None) + ] + + +def test_gemini_endpoints_list_takes_priority_over_api_key_list() -> None: + section = GeminiProviderConfig( + api_key_list=["k1", "k2"], + endpoints=[ProviderEndpoint(label="only", api_key="sk-endpoint")], + ) + + resolved = provider_endpoints(section) + + assert [e.label for e in resolved] == ["only"] + assert [e.api_key for e in resolved] == ["sk-endpoint"] From 98b736a38ad1c1fe3c87079935a10ab637bf7d0e Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 00:17:47 +0800 Subject: [PATCH 19/78] fix(providers): resolve the construction-time window without the network The window ladder put a synchronous OpenRouter catalogue fetch (10s httpx timeout on a cold cache) inside AgentLoop construction and the /model switch handler -- the former stalls startup, the latter runs on the asyncio event loop and freezes every turn with it. Both callers now resolve with allow_fetch=False: an in-process cache of any age answers, then the on-disk cache of any age, then the documented fallback, and the network is never touched. The per-call usage path keeps refreshing normally; it already lives inside an await and is where a stale window catches up. Co-authored-by: Claude (claude-fable-5) --- raven/agent/loop/main.py | 13 +++- raven/providers/rates.py | 46 +++++++++++--- tests/test_agent_loop_usage_sink.py | 77 +++++++++++++++++++++++- tests/test_provider_rates.py | 93 ++++++++++++++++++++++++++++- 4 files changed, 215 insertions(+), 14 deletions(-) diff --git a/raven/agent/loop/main.py b/raven/agent/loop/main.py index 7ebb6e39..3892d4ff 100644 --- a/raven/agent/loop/main.py +++ b/raven/agent/loop/main.py @@ -381,7 +381,13 @@ def __init__( # A caller that passed a positive value pinned the window; None/0 means # "figure it out", resolved once here against the model's real window. self._context_window_pinned = bool(context_window_tokens) - self.context_window_tokens = context_window_tokens or effective_context_window(self.model, None) + # allow_fetch=False: construction must not block on a synchronous + # network call for an OpenRouter model's window -- whatever is already + # cached (in-process or on disk, any age) answers instead. See + # rates._fetch_openrouter_models. + self.context_window_tokens = context_window_tokens or effective_context_window( + self.model, None, allow_fetch=False + ) self.brave_api_key = brave_api_key self.jina_api_key = jina_api_key self.web_proxy = web_proxy @@ -707,7 +713,10 @@ def refresh_context_window(self) -> None: """ if self._context_window_pinned: return - self.context_window_tokens = effective_context_window(self.model, None) + # allow_fetch=False: a /model switch runs inside the running event + # loop, so this must not block it on a synchronous network call. See + # rates._fetch_openrouter_models. + self.context_window_tokens = effective_context_window(self.model, None, allow_fetch=False) def _register_default_tools(self) -> None: """Register the default set of tools.""" diff --git a/raven/providers/rates.py b/raven/providers/rates.py index 0f5abb3b..82c150ce 100644 --- a/raven/providers/rates.py +++ b/raven/providers/rates.py @@ -184,15 +184,36 @@ def _try_litellm_rates(model: str, input_tokens: int, output_tokens: int) -> tup return None -def _fetch_openrouter_models() -> dict[str, dict]: +def _fetch_openrouter_models(*, allow_fetch: bool = True) -> dict[str, dict]: """Return OpenRouter's model table, fetched live and cached 1h in-process. Each entry is ``{"pricing": ..., "context_length": ...}``, keyed by the full id. On any network failure, returns the stale cache (or an empty dict) -- pricing must never raise into the cost path. + + ``allow_fetch=False`` is for a caller inside object construction or an + asyncio event loop -- ``AgentLoop.__init__`` and a ``/model`` switch, both of + which resolve a context window before there is a request to size. Those + callers want whatever is already on hand: an in-process cache of any age + answers, then an on-disk cache of any age, then an empty table -- the + network is never touched, because a synchronous ``httpx.Client`` there would + block startup or freeze the running event loop for up to 10s. A stale + answer only costs a stale window; a blocked event loop costs the whole + turn. The per-call usage path is the place that still refreshes normally -- + it already runs inside an ``await``, and is where a stale price or window + is supposed to catch up. """ global _OPENROUTER_CACHE, _OPENROUTER_CACHE_TIME + if not allow_fetch: + if _OPENROUTER_CACHE: + return _OPENROUTER_CACHE + disk = model_catalog_cache.load() + if disk is not None: + _OPENROUTER_CACHE, _OPENROUTER_CACHE_TIME = disk + return _OPENROUTER_CACHE + return {} + now = time.time() if _OPENROUTER_CACHE and (now - _OPENROUTER_CACHE_TIME) < _OPENROUTER_CACHE_TTL: return _OPENROUTER_CACHE @@ -354,7 +375,7 @@ def openrouter_input_modalities(model: str) -> tuple[str, ...] | None: return tuple(mods) if isinstance(mods, list) and mods else None -def _lookup_openrouter_entry(model: str) -> dict | None: +def _lookup_openrouter_entry(model: str, *, allow_fetch: bool = True) -> dict | None: """This model's row in OpenRouter's catalogue, or None. Only for ids that name OpenRouter. The table was once consulted for every id, @@ -364,11 +385,16 @@ def _lookup_openrouter_entry(model: str) -> dict | None: this table about a request that does not go to OpenRouter -- not the bare alias, which stays because within OpenRouter's own namespace a bare id names the same model the full one does. + + ``allow_fetch`` passes straight through to ``_fetch_openrouter_models``; see + there for what it changes. Called with the default omitted rather than + ``allow_fetch=True`` explicitly, so a test double standing in for the fetch + with the old zero-argument signature still works unchanged. """ if not model.startswith("openrouter/"): return None key = model.removeprefix("openrouter/") - table = _fetch_openrouter_models() + table = _fetch_openrouter_models() if allow_fetch else _fetch_openrouter_models(allow_fetch=False) entry = table.get(key) if entry is None and "/" in key: entry = table.get(key.split("/", 1)[1]) @@ -474,7 +500,7 @@ def _try_litellm_context_window(model: str) -> int | None: return None -def resolve_context_window(model: str) -> int | None: +def resolve_context_window(model: str, *, allow_fetch: bool = True) -> int | None: """Return a model's real context window in tokens, or None. LiteLLM's static metadata first, then OpenRouter's catalogue for ids that @@ -482,12 +508,15 @@ def resolve_context_window(model: str) -> int | None: trimming, so a community-maintained file that goes stale or wrong would shape the next request rather than cost a label. Unknown models return None so the caller keeps its configured default. + + ``allow_fetch=False`` passes straight through to the OpenRouter tier; see + ``_fetch_openrouter_models`` for what it changes. """ window = _try_litellm_context_window(model) if window: return window - entry = _lookup_openrouter_entry(model) + entry = _lookup_openrouter_entry(model, allow_fetch=allow_fetch) if entry: try: length = int(entry.get("context_length") or 0) @@ -498,7 +527,7 @@ def resolve_context_window(model: str) -> int | None: return None -def effective_context_window(model: str, configured: int | None) -> int: +def effective_context_window(model: str, configured: int | None, *, allow_fetch: bool = True) -> int: """The context window to size trimming with -- the decision ladder's front door. Explicit configuration wins outright: a user or caller who pinned a number @@ -511,10 +540,13 @@ def effective_context_window(model: str, configured: int | None) -> int: ``resolve_context_window`` already folds every LiteLLM and network failure into ``None`` rather than raising (see its tiers), so there is no exception left here to catch. + + ``allow_fetch=False`` passes straight through to ``resolve_context_window``; + see ``_fetch_openrouter_models`` for what it changes. """ if configured: return configured - return resolve_context_window(model) or DEFAULT_CONTEXT_WINDOW_TOKENS + return resolve_context_window(model, allow_fetch=allow_fetch) or DEFAULT_CONTEXT_WINDOW_TOKENS def reset_openrouter_cache() -> None: diff --git a/tests/test_agent_loop_usage_sink.py b/tests/test_agent_loop_usage_sink.py index e9e716f8..2d5e3ac1 100644 --- a/tests/test_agent_loop_usage_sink.py +++ b/tests/test_agent_loop_usage_sink.py @@ -10,6 +10,7 @@ import json import tempfile +import time from pathlib import Path import httpx @@ -219,13 +220,85 @@ def test_refresh_context_window_is_a_noop_once_pinned(workspace, monkeypatch): def test_refresh_context_window_follows_the_new_model_when_unpinned(workspace, monkeypatch): - """Unpinned, a ``/model`` switch re-walks the ladder for the new model.""" + """Unpinned, a ``/model`` switch re-walks the ladder for the new model. + + The switch runs inside the running event loop, so the ladder is walked with + ``allow_fetch=False`` -- an in-process cache entry of any age answers rather + than a live fetch. Populated directly rather than through a mocked network + call, which ``allow_fetch=False`` never reaches. + """ provider = UsageProvider("stub", 0, 0) agent = _make_agent(workspace, provider, model="stub", window=None) assert agent.context_window_tokens == rates.DEFAULT_CONTEXT_WINDOW_TOKENS - _patch_live_openrouter_window(monkeypatch, 163840) + # conftest's autouse guard stubs the fetch to a zero-argument lambda; restore + # the real one so allow_fetch=False's in-process-cache branch actually runs. + monkeypatch.setattr(rates, "_fetch_openrouter_models", _REAL_FETCH) + rates._OPENROUTER_CACHE["deepseek/deepseek-v4-pro"] = { + "pricing": {"prompt": "0.0000005", "completion": "0.0000015"}, + "context_length": 163840, + } + monkeypatch.setattr(rates, "_OPENROUTER_CACHE_TIME", time.time() - rates._OPENROUTER_CACHE_TTL - 1000) agent.model = "openrouter/deepseek/deepseek-v4-pro" agent.refresh_context_window() assert agent.context_window_tokens == 163840 + + +# --------------------------------------------------------------------------- # +# allow_fetch=False: construction and refresh never touch the network # +# --------------------------------------------------------------------------- # + + +def _forbid_network_client(monkeypatch): + """Restore the real fetch, then count real ``httpx.Client`` builds. + + Not a raise: ``_fetch_openrouter_models`` wraps the fetch in + ``except Exception`` to degrade on a network failure, so a raise from here + would be swallowed as "the network failed" and the test would pass for the + wrong reason. A counter the caller asserts is 0 actually distinguishes + "never touched the network" from "touched it and degraded". + """ + counter = {"calls": 0} + real_client = rates.httpx.Client + + def _counting_client(*args, **kwargs): + counter["calls"] += 1 + return real_client(*args, **kwargs) + + monkeypatch.setattr(rates, "_fetch_openrouter_models", _REAL_FETCH) + monkeypatch.setattr(rates.httpx, "Client", _counting_client) + return counter + + +def test_construction_on_an_openrouter_model_never_touches_the_network(workspace, monkeypatch, tmp_path): + """The regression this fixes: constructing on an unmapped OpenRouter model + used to fetch synchronously, blocking startup for up to 10s.""" + from raven.providers import model_catalog_cache + + monkeypatch.setattr(model_catalog_cache, "_CACHE_PATH", tmp_path / "model-catalog.json", raising=False) + counter = _forbid_network_client(monkeypatch) + + provider = UsageProvider("openrouter/deepseek/deepseek-v4-pro", 0, 0) + agent = _make_agent(workspace, provider, model="openrouter/deepseek/deepseek-v4-pro", window=None) + + assert agent.context_window_tokens == rates.DEFAULT_CONTEXT_WINDOW_TOKENS + assert counter["calls"] == 0 + + +def test_refresh_context_window_on_an_openrouter_model_never_touches_the_network(workspace, monkeypatch, tmp_path): + """The other half: a ``/model`` switch inside the running event loop must not + freeze it on a synchronous fetch either.""" + from raven.providers import model_catalog_cache + + monkeypatch.setattr(model_catalog_cache, "_CACHE_PATH", tmp_path / "model-catalog.json", raising=False) + + provider = UsageProvider("stub", 0, 0) + agent = _make_agent(workspace, provider, model="stub", window=None) + + counter = _forbid_network_client(monkeypatch) + agent.model = "openrouter/deepseek/deepseek-v4-pro" + agent.refresh_context_window() + + assert agent.context_window_tokens == rates.DEFAULT_CONTEXT_WINDOW_TOKENS + assert counter["calls"] == 0 diff --git a/tests/test_provider_rates.py b/tests/test_provider_rates.py index be32b14d..8274e6d1 100644 --- a/tests/test_provider_rates.py +++ b/tests/test_provider_rates.py @@ -306,24 +306,38 @@ def test_a_window_unknown_to_every_source_is_none(monkeypatch): def test_a_configured_window_wins_even_when_the_real_one_disagrees(monkeypatch): """A pin is an override, so it answers even when the model has a real window.""" - monkeypatch.setattr(rates, "resolve_context_window", lambda model: 200_000) + monkeypatch.setattr(rates, "resolve_context_window", lambda model, **kw: 200_000) assert rates.effective_context_window("anthropic/claude-sonnet-4-5", 40_000) == 40_000 def test_no_configured_window_falls_back_to_the_real_one(monkeypatch): - monkeypatch.setattr(rates, "resolve_context_window", lambda model: 200_000) + monkeypatch.setattr(rates, "resolve_context_window", lambda model, **kw: 200_000) assert rates.effective_context_window("anthropic/claude-sonnet-4-5", None) == 200_000 assert rates.effective_context_window("anthropic/claude-sonnet-4-5", 0) == 200_000 def test_neither_configured_nor_resolvable_uses_the_documented_default(monkeypatch): - monkeypatch.setattr(rates, "resolve_context_window", lambda model: None) + monkeypatch.setattr(rates, "resolve_context_window", lambda model, **kw: None) assert rates.effective_context_window("some/unknown-model", None) == rates.DEFAULT_CONTEXT_WINDOW_TOKENS +def test_effective_context_window_passes_allow_fetch_through(monkeypatch): + """The construction-time caller's ``allow_fetch=False`` must reach the ladder.""" + seen: dict = {} + monkeypatch.setattr( + rates, + "resolve_context_window", + lambda model, **kw: seen.update(kw) or 200_000, + ) + + rates.effective_context_window("anthropic/claude-sonnet-4-5", None, allow_fetch=False) + + assert seen == {"allow_fetch": False} + + # --- Models whose driver would start an interactive login --- _COPILOT_MODELS = [ @@ -650,3 +664,76 @@ def test_disk_write_is_atomic(monkeypatch, disk_cache): assert list(disk_cache.parent.glob("*.tmp")) == [] json.loads(disk_cache.read_text(encoding="utf-8")) + + +# --- allow_fetch=False: construction / event-loop callers never touch the network --- +# +# ``AgentLoop.__init__`` and ``refresh_context_window`` resolve a window before +# there is a request to size, one of them inside the running event loop -- a +# synchronous ``httpx.Client`` there is a stall, not a slow answer. Both pass +# ``allow_fetch=False`` down to the fetch, which is asserted here to never build +# a client: whatever is cached, of any age, answers instead. + + +def _forbid_network_client(monkeypatch): + """Restore the real fetch, then count real ``httpx.Client`` builds. + + Not a raise: ``_fetch_openrouter_models`` wraps the fetch in + ``except Exception`` to degrade on a network failure, so a raise from here + would be swallowed as "the network failed" and the test would pass for the + wrong reason. A counter the caller asserts is 0 actually distinguishes + "never touched the network" from "touched it and degraded". + """ + counter = {"calls": 0} + real_client = rates.httpx.Client + + def _counting_client(*args, **kwargs): + counter["calls"] += 1 + return real_client(*args, **kwargs) + + monkeypatch.setattr(rates, "_fetch_openrouter_models", _REAL_FETCH) + monkeypatch.setattr(rates.httpx, "Client", _counting_client) + return counter + + +def test_allow_fetch_false_serves_an_expired_in_memory_cache(monkeypatch): + """In-process cache of any age answers -- expired is still better than a stall.""" + counter = _forbid_network_client(monkeypatch) + rates._OPENROUTER_CACHE.update( + { + "deepseek/deepseek-v4-pro": { + "pricing": {"prompt": "0.0000005", "completion": "0.0000015"}, + "context_length": 163840, + } + } + ) + monkeypatch.setattr(rates, "_OPENROUTER_CACHE_TIME", time.time() - rates._OPENROUTER_CACHE_TTL - 1000) + + assert rates._fetch_openrouter_models(allow_fetch=False) is rates._OPENROUTER_CACHE + assert resolve_context_window("openrouter/deepseek/deepseek-v4-pro", allow_fetch=False) == 163840 + assert counter["calls"] == 0 + + +def test_allow_fetch_false_falls_back_to_an_expired_disk_cache(monkeypatch, disk_cache): + """Empty in-memory, a stale disk file answers and refills memory -- never network.""" + counter = _forbid_network_client(monkeypatch) + stale_at = time.time() - (rates._OPENROUTER_CACHE_TTL + 100) + disk_cache.write_text(json.dumps(_disk_payload(stale_at)), encoding="utf-8") + + assert resolve_context_window("openrouter/deepseek/deepseek-v4-pro", allow_fetch=False) == 163840 + assert rates._OPENROUTER_CACHE, "the disk hit should have refilled the in-process cache" + assert counter["calls"] == 0 + + +def test_allow_fetch_false_with_nothing_cached_is_none_and_never_hits_the_network(monkeypatch, disk_cache): + """Cold everywhere: {} / None, never a synchronous request that could stall + construction or the event loop for up to 10s.""" + counter = _forbid_network_client(monkeypatch) + + assert rates._fetch_openrouter_models(allow_fetch=False) == {} + assert resolve_context_window("openrouter/deepseek/deepseek-v4-pro", allow_fetch=False) is None + assert ( + rates.effective_context_window("openrouter/deepseek/deepseek-v4-pro", None, allow_fetch=False) + == rates.DEFAULT_CONTEXT_WINDOW_TOKENS + ) + assert counter["calls"] == 0 From 49762654c8cf7838858545f12393807a34f94768 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 00:24:15 +0800 Subject: [PATCH 20/78] refactor(cli): move the wizard's everos cluster into its own module onboard_commands.py had grown past 5000 lines; the EverOS/Step4 cluster (~1500 lines, 36 symbols) now lives in onboard_everos.py. Pure move, zero behavior change: shared wizard UI state stays in onboard_commands and both modules reach across through module references with call-time attribute access, so import order cannot bite and monkeypatches keep hitting the code that actually runs. Every migrated patch target in the test file was mutation-checked one by one against pointing at the old module -- all 23 turn red, none patch a shell. Co-authored-by: Claude (claude-fable-5) --- raven/cli/onboard_commands.py | 1523 +-------------------------- raven/cli/onboard_everos.py | 1525 ++++++++++++++++++++++++++++ tests/test_cli_onboard_commands.py | 110 +- 3 files changed, 1589 insertions(+), 1569 deletions(-) create mode 100644 raven/cli/onboard_everos.py diff --git a/raven/cli/onboard_commands.py b/raven/cli/onboard_commands.py index a6d3701a..4701929c 100644 --- a/raven/cli/onboard_commands.py +++ b/raven/cli/onboard_commands.py @@ -36,6 +36,7 @@ from rich.console import Console from rich.panel import Panel +from raven.cli import onboard_everos from raven.cli._helpers import ( DEFAULT_PROBE_MESSAGE, print_probe_troubleshooting, @@ -440,7 +441,7 @@ def _bootstrap_empty_config() -> None: path = get_config_path() if not path.exists(): save_config(load_config()) # writes default Config() to disk - _init_extension_block_defaults() + onboard_everos._init_extension_block_defaults() workspace = get_workspace_path() workspace.mkdir(parents=True, exist_ok=True) sync_workspace_templates(workspace) @@ -2691,1516 +2692,6 @@ def _step3_channel(*, channel: Optional[str], skip: bool, non_interactive: bool) _manage_existing_channels() -# --------------------------------------------------------------------------- -# Step 4 — EverOS long-term memory -# --------------------------------------------------------------------------- - - -def _set_memory_backend(backend: Optional[str]) -> None: - """Set ``memory.backend`` (``"everos"`` / ``None``) via the ops layer.""" - from raven.config.update import set_memory_backend - - set_memory_backend(backend) - - -def _init_extension_block_defaults() -> None: - """Seed the memory / plugins / skillForge extension defaults via the ops layer.""" - from raven.config.update import init_extension_block_defaults - - init_extension_block_defaults() - - -def _everos_section(section: str) -> dict[str, Any]: - from raven.config.update_everos import everos_section - - return everos_section(section) - - -def _everos_role_configured(section: str) -> bool: - from raven.config.update_everos import everos_role_configured - - return everos_role_configured(section) - - -def _memory_enabled() -> bool: - """True iff EverOS memory is both selected AND usable on disk. - - "Usable" means the llm role is configured -- that is the whole requirement. - embedding is advised but optional: without it the adapter searches lexically - instead of semantically, which is weaker memory rather than none, and gating - on it here would tell a user who skipped it that memory is off (and skip the - import step along with it). - """ - data = _load_raw_config() - if (data.get("memory") or {}).get("backend") != "everos": - return False - return _everos_role_configured("llm") - - -# Providers whose main model can be reused as the EverOS memory LLM: they -# speak the OpenAI chat-completions protocol that EverOS's bare OpenAI client -# requires. OAuth providers (github_copilot / openai_codex) and non-OpenAI -# wire protocols (anthropic / gemini) are excluded. -_OPENAI_COMPATIBLE_PROVIDERS = {"openrouter", "openai", "deepseek", "custom"} - -# Fallback OpenAI-compatible base URLs for providers whose registry -# ``default_api_base`` is empty (they rely on the SDK's built-in default, -# which EverOS's bare client doesn't know). EverOS needs an explicit base_url. -_PROVIDER_BASE_URL_FALLBACK = { - "openai": "https://api.openai.com/v1", - "deepseek": "https://api.deepseek.com/v1", - "openrouter": "https://openrouter.ai/api/v1", -} - - -def _resolve_model_provider(model: str) -> Optional[str]: - """Best-effort: which configured provider does ``model`` belong to? - - Prefixed models (``openrouter/...`` / ``openai/gpt-4o``) read off the head. - A custom endpoint stores its model as a BARE id (e.g. ``qwen-max``) with no - prefix, so an unrecognized head falls back to ``"custom"`` when a custom - provider is actually configured with a key. Returns ``None`` when no match. - """ - from raven.providers.registry import split_model_id - - if not model: - return None - head, _ = split_model_id(model) - if head: - from raven.config.update_providers import provider_field_specs - - try: - provider_field_specs(head) - return head - except KeyError: - pass - # No usable prefix → could be a bare custom-endpoint model. - from raven.config.schema import ProviderConfig - from raven.providers.auth import credential_status - - custom = (_load_raw_config().get("providers") or {}).get("custom") or {} - if credential_status("custom", ProviderConfig.model_validate(custom)).ok: - return "custom" - # A bare id that still matches a known provider head (rare; e.g. a direct - # provider's bare default before prefixing) — accept the head if known. - return head if head in _OPENAI_COMPATIBLE_PROVIDERS else None - - -def _model_is_openai_compatible(model: Optional[str]) -> bool: - """Heuristic: can the main chat model's provider be reused for memory LLM? - - EverOS's memory LLM uses a bare OpenAI client, so the main model is - reusable only when its provider speaks the OpenAI chat protocol. Custom - endpoints are OpenAI-compatible by definition (the wizard only offers - ``custom`` for OpenAI-compatible endpoints). - """ - if not model: - return False - return _resolve_model_provider(model) in _OPENAI_COMPATIBLE_PROVIDERS - - -def _resolve_reuse_llm_creds(main_model: str) -> dict[str, Optional[str]]: - """Map a litellm-style main model to bare EverOS LLM settings. - - EverOS sends ``EVEROS_LLM__MODEL`` to ``base_url`` via a bare OpenAI - client, so: - - strip the provider's litellm prefix to the bare model id the upstream - endpoint expects (``openrouter/anthropic/claude-x`` → ``anthropic/claude-x``; - a custom endpoint's bare id is used as-is); - - resolve the provider's real ``base_url`` (configured ``apiBase`` → - registry ``default_api_base`` → a known fallback); - - carry the provider's stored api_key. - """ - from raven.providers.registry import find_by_name, normalize_provider_name, split_model_id - - provider = _resolve_model_provider(main_model) or split_model_id(main_model)[0] - spec = find_by_name(provider) - # Through the ops library, so a section still stored under the provider's - # pre-rename name is found -- a raw lookup by the resolved name is not. - from raven.config.update_providers import get_provider_config - - # No `if spec` gate: LiteLLM-only vendors have no spec of ours yet their - # section holds real credentials, and gating on the spec silently handed the - # probe an empty api_key while the main model was working fine. - try: - _resolved = get_provider_config(provider, redact_secrets=False) - except KeyError: - _resolved = {} - prov_cfg = {"apiKey": _resolved.get("api_key"), "apiBase": _resolved.get("api_base")} if _resolved else {} - - # Strip the routing prefix to the bare model id the upstream endpoint - # expects: litellm consumes it, the raw OpenAI client must not see it. Only - # a prefix naming this provider is stripped -- a custom endpoint stores a - # bare id already, and anything else is part of the vendor's own model id. - bare_model = main_model - head, rest = split_model_id(main_model) - known_prefixes = set(spec.route_names) if spec else {normalize_provider_name(provider)} - if spec: - known_prefixes.add(normalize_provider_name(spec.model_prefix)) - if head and head in known_prefixes: - bare_model = rest - - base_url = ( - prov_cfg.get("apiBase") - or (getattr(spec, "default_api_base", "") if spec else "") - or _PROVIDER_BASE_URL_FALLBACK.get(provider) - ) - return { - "model": bare_model, - "api_key": prov_cfg.get("apiKey"), - "base_url": base_url, - } - - -def _prompt_text(label: str, *, secret: bool = False, default: str = "", allow_back: bool = False) -> Any: - """Prompt for free text. With ``allow_back``, an empty submit returns - ``_BACK`` (and a hint is shown); otherwise returns the stripped string.""" - questionary = _require_questionary() - from raven.cli._styles import RAVEN_STYLE - - placeholder = _back_placeholder(allow_back) - if secret: - value = questionary.password(label, placeholder=placeholder, style=RAVEN_STYLE, qmark=_QMARK).ask() - else: - value = questionary.text(label, default=default, placeholder=placeholder, style=RAVEN_STYLE, qmark=_QMARK).ask() - if value is None: - raise typer.Exit(1) - value = value.strip() - if allow_back and value == "": - return _BACK - return value - - -def _probe_everos_chat(model: Optional[str], *, api_key: Optional[str], base_url: Optional[str]) -> tuple[bool, str]: - """Real capability probe for a memory-LLM endpoint: ``POST - {base_url}/chat/completions`` once and confirm a choice comes back. Unlike a - bare ``GET /models`` connectivity check, this exercises the picked model, so - an endpoint that lists models but doesn't serve the chosen id fails here - instead of reporting a false green. Provider-agnostic; never raises.""" - import httpx - - if not base_url: - return False, "no base_url configured" - url = base_url.rstrip("/") + ("/chat/completions" if "/v1" in base_url else "/v1/chat/completions") - headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} - body = {"model": model, "messages": [{"role": "user", "content": "ping"}], "max_tokens": 1} - try: - with httpx.Client(timeout=15) as client: - resp = client.post(url, headers=headers, json=body) - if resp.status_code != 200: - return False, f"HTTP {resp.status_code}: {resp.text[:200]}" - data = resp.json() - except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc: - return False, f"probe failed: {exc}" - choices = data.get("choices") if isinstance(data, dict) else None - if isinstance(choices, list) and choices and isinstance(choices[0], dict): - return True, "ok" - return False, "endpoint returned no completion" - - -def _verify_everos_llm( - label: str, - *, - model: Optional[str], - api_key: Optional[str], - base_url: Optional[str], - non_interactive: bool, - warnings: list[str], - continue_hint: Optional[tuple[str, str]] = None, -) -> bool: - """Probe the memory LLM with a real chat completion, offering retry/continue on failure.""" - console.print(_t(f" [dim]⏳ Verifying {label}…[/dim]", f" [dim]⏳ 正在验证 {label}…[/dim]")) - ok, detail = _probe_everos_chat(model, api_key=api_key, base_url=base_url) - if ok: - console.print(_t(f" [green]✓ {label} connected.[/green]", f" [green]✓ {label} 连接成功。[/green]")) - return True - console.print( - _t( - f" [yellow]✗ Couldn't verify {label}: {detail}[/yellow]", - f" [yellow]✗ 验证失败 {label}:{detail}[/yellow]", - ) - ) - if continue_hint: - cont_label = _t(f"Continue anyway ({continue_hint[0]})", f"仍然继续({continue_hint[1]})") - else: - cont_label = _t("Continue anyway", "仍然继续") - choice = _failure_choice( - [ - (_t("Re-enter", "重新填写"), "rekey"), - (cont_label, "continue"), - ], - non_interactive=non_interactive, - ) - if choice == "rekey": - return False - warnings.append(label) - return True - - -def _verify_rerank( - label: str, - *, - model: Optional[str], - api_key: Optional[str], - base_url: Optional[str], - rerank_provider: Optional[str], - non_interactive: bool, - warnings: list[str], - continue_hint: Optional[tuple[str, str]] = None, -) -> bool: - """Probe a rerank endpoint with a provider-specific request, offering retry/continue on failure.""" - console.print(_t(f" [dim]⏳ Verifying {label}…[/dim]", f" [dim]⏳ 正在验证 {label}…[/dim]")) - ok, detail = _probe_rerank(model, api_key=api_key, base_url=base_url, rerank_provider=rerank_provider) - if ok: - console.print(_t(f" [green]✓ {label} connected.[/green]", f" [green]✓ {label} 连接成功。[/green]")) - return True - console.print( - _t( - f" [yellow]✗ Couldn't verify {label}: {detail}[/yellow]", - f" [yellow]✗ 验证失败 {label}:{detail}[/yellow]", - ) - ) - if continue_hint: - cont_label = _t(f"Continue anyway ({continue_hint[0]})", f"仍然继续({continue_hint[1]})") - else: - cont_label = _t("Continue anyway", "仍然继续") - choice = _failure_choice( - [ - (_t("Re-enter", "重新填写"), "rekey"), - (cont_label, "continue"), - ], - non_interactive=non_interactive, - ) - if choice == "rekey": - return False - warnings.append(label) - return True - - -def _probe_rerank( - model: Optional[str], - *, - api_key: Optional[str], - base_url: Optional[str], - rerank_provider: Optional[str], -) -> tuple[bool, str]: - """Real capability probe for a rerank endpoint. Dispatches by provider - protocol (vllm / deepinfra / dashscope). Never raises.""" - import httpx - - if not base_url or not model: - return False, "no base_url or model configured" - headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} - headers["Content-Type"] = "application/json" - - try: - if rerank_provider == "deepinfra": - url = f"{base_url.rstrip('/')}/{model}" - body: dict = {"queries": ["ping"], "documents": ["pong"]} - elif rerank_provider == "dashscope": - url = f"{base_url.rstrip('/')}/api/v1/services/rerank/text-rerank/text-rerank" - body = { - "model": model, - "input": {"query": "ping", "documents": ["pong"]}, - "parameters": {"return_documents": False, "top_n": 1}, - } - else: # vllm / OpenAI-compat - url = f"{base_url.rstrip('/')}/rerank" - body = {"model": model, "query": "ping", "documents": ["pong"]} - - with httpx.Client(timeout=15) as client: - resp = client.post(url, json=body, headers=headers) - if resp.status_code != 200: - return False, f"HTTP {resp.status_code}: {resp.text[:200]}" - data = resp.json() - except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc: - return False, f"probe failed: {exc}" - - if rerank_provider == "deepinfra": - scores = data.get("scores") - if isinstance(scores, list) and scores: - return True, "ok" - return False, "endpoint returned no scores" - if rerank_provider == "dashscope": - output = data.get("output") - results = output.get("results") if isinstance(output, dict) else None - if isinstance(results, list) and results: - return True, "ok" - return False, "endpoint returned no results" - # vllm - results = data.get("results") - if isinstance(results, list) and results: - return True, "ok" - return False, "endpoint returned no results" - - -_REQUIRED_EMBEDDING_DIM = 1024 - - -def _probe_embedding_dim(url: str, headers: dict, model: str) -> int | str: - """Try embedding with ``dimensions=1024``; fall back to native dim. - - Returns the effective dimension (int) on success, or an error - description (str) on failure. - """ - import httpx - - def _try_embed(client: httpx.Client, body: dict) -> int | str: - try: - resp = client.post(url, json=body, headers=headers) - if resp.status_code != 200: - return f"HTTP {resp.status_code}" - items = resp.json().get("data", []) - if not items: - return "empty response" - first = items[0] - if not isinstance(first, dict): - return "unexpected response format" - return len(first.get("embedding", [])) - except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc: - return str(exc) - - with httpx.Client(timeout=15) as client: - result = _try_embed( - client, {"model": model, "input": ["dimension check"], "dimensions": _REQUIRED_EMBEDDING_DIM} - ) - if result == _REQUIRED_EMBEDDING_DIM: - return result - return _try_embed(client, {"model": model, "input": ["dimension check"]}) - - -def _verify_embedding_dim( - *, - model: Optional[str], - api_key: Optional[str], - base_url: Optional[str], - non_interactive: bool, -) -> bool: - """Send a test embedding request and verify the vector dimension is 1024. - - Returns True to proceed, False to re-prompt. - """ - if not base_url or not model: - return True - - url = base_url.rstrip("/") + "/embeddings" - headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} - - while True: - console.print( - _t( - " [dim]⏳ Checking embedding dimension…[/dim]", - " [dim]⏳ 正在检测 embedding 维度…[/dim]", - ) - ) - result = _probe_embedding_dim(url, headers, model) - - if result == _REQUIRED_EMBEDDING_DIM: - console.print( - _t( - f" [green]✓ Supports {result}-dim.[/green]", - f" [green]✓ 支持 {result} 维。[/green]", - ) - ) - return True - - if isinstance(result, int) and result < _REQUIRED_EMBEDDING_DIM: - console.print( - _t( - f" [red]✗ Dimension too small: model outputs {result}-dim, " - f"EverOS requires >= {_REQUIRED_EMBEDDING_DIM}. Please pick another model.[/red]", - f" [red]✗ 维度不足：模型输出 {result} 维，" - f"EverOS 要求 >= {_REQUIRED_EMBEDDING_DIM} 维，请重新选择。[/red]", - ) - ) - return False - - if isinstance(result, int) and result > _REQUIRED_EMBEDDING_DIM: - console.print( - _t( - f" [red]✗ Model outputs {result}-dim and does not support the " - f"dimensions parameter to truncate to {_REQUIRED_EMBEDDING_DIM}. " - "Please pick another model.[/red]", - f" [red]✗ 模型输出 {result} 维，且不支持 dimensions 参数" - f"截断到 {_REQUIRED_EMBEDDING_DIM} 维，请重新选择。[/red]", - ) - ) - return False - - console.print( - _t( - f" [yellow]✗ Couldn't verify dimension: {result}[/yellow]", - f" [yellow]✗ 无法验证维度：{result}[/yellow]", - ) - ) - if non_interactive: - return False - choice = _failure_choice( - [ - (_t("Retry", "重试"), "retry"), - (_t("Re-enter", "重新选择"), "rekey"), - ], - non_interactive=False, - ) - if choice == "rekey": - return False - - -# Curated OpenAI-compatible endpoints for EverOS memory models. Picking one -# pre-fills its base_url (mirrors the main provider step); everything else is -# reachable via "reuse an existing endpoint" or "custom" (type a base_url). -# These are the providers' documented OpenAI-compatible /v1 endpoints. -_EVEROS_PROVIDERS: list[dict[str, Any]] = [ - { - "name": "openai", - "label": "OpenAI", - "label_zh": "OpenAI", - "base_url": "https://api.openai.com/v1", - "supports": {"llm", "embedding", "multimodal"}, - }, - { - "name": "openrouter", - "label": "OpenRouter", - "label_zh": "OpenRouter", - "base_url": "https://openrouter.ai/api/v1", - "supports": {"llm", "embedding", "rerank", "multimodal"}, - "rerank_provider": "vllm", - }, - { - "name": "deepseek", - "label": "DeepSeek", - "label_zh": "DeepSeek", - "base_url": "https://api.deepseek.com/v1", - "supports": {"llm"}, - }, - { - "name": "deepinfra", - "label": "DeepInfra", - "label_zh": "DeepInfra", - "base_url": "https://api.deepinfra.com/v1/openai", - "supports": {"llm", "embedding", "rerank"}, - "rerank_provider": "deepinfra", - "rerank_base_url": "https://api.deepinfra.com/v1/inference", - }, - { - "name": "siliconflow", - "label": "SiliconFlow", - "label_zh": "硅基流动 SiliconFlow", - "base_url": "https://api.siliconflow.cn/v1", - "supports": {"llm", "embedding", "rerank"}, - "rerank_provider": "vllm", - }, - { - "name": "dashscope", - "label": "DashScope (Alibaba)", - "label_zh": "阿里百炼 DashScope", - "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", - "supports": {"llm", "embedding", "rerank"}, - "rerank_provider": "dashscope", - "rerank_base_url": "https://dashscope.aliyuncs.com", - }, -] - - -def _match_provider_by_url(base_url: Optional[str]) -> Optional[str]: - """Reverse-lookup a curated provider name from its base_url.""" - if not base_url: - return None - normalized = base_url.rstrip("/") - for prov in _EVEROS_PROVIDERS: - if prov["base_url"].rstrip("/") == normalized: - return prov["name"] - return None - - -# Per-role config: menu/verify label, model-id example, whether optional, and -# whether to run a connectivity probe after configuring (rerank/multimodal use -# non-chat endpoints whose /models probe isn't a reliable health check). -_EVEROS_ROLES: dict[str, dict[str, Any]] = { - "llm": { - "label": ("Memory LLM", "记忆 LLM"), - "example": "gpt-4.1-mini", - "optional": False, - "verify": True, - "purpose": ( - "Reads each conversation to judge what matters and extract the key points.", - "从对话中判断信息边界、抽取要点。", - ), - # Worded as a floor rather than a default: the field is pre-filled with - # the user's own main model, because a recommended id is only reachable - # if their key carries it. This tells them how to judge their own. - "recommendation": ( - "Capability floor: [bold]gpt-4.1-mini[/bold] -- weaker models degrade extraction", - "能力下限参考 [bold]gpt-4.1-mini[/bold]：低于这个水平会明显影响提取质量", - ), - "continue_hint": ("memory extraction may fail", "记忆抽取可能失败"), - }, - "embedding": { - "label": ("Memory embedding", "记忆 embedding"), - "example": "Qwen/Qwen3-Embedding-4B", - # Optional in the sense that memory still functions without it: the - # adapter drops to KEYWORD search, which needs no vectors. Strongly - # advised all the same -- lexical recall misses a memory the moment the - # user phrases the question differently. - "optional": True, - "verify": True, - "purpose": ( - "Turns text into vectors so memories are found by meaning, not just keywords.", - "把文字转成向量，让记忆能按「意思」检索，而不只是按关键词。", - ), - "tag": ( - "[accent](optional, strongly advised)[/accent]", - "[accent]（可选，强烈建议配置）[/accent]", - ), - "cost": ( - "Without it: rephrase a question and it may miss a memory you have;\n recall can only match keywords.", - "不配置：换个说法提问就可能找不到已有记忆，记忆召回时只能使用关键词检索。", - ), - "recommendation": ( - "Recommended: [bold]Qwen/Qwen3-Embedding-4B[/bold] -- must be [bold yellow]1024-dim[/bold yellow],\n" - " Chinese + English", - "推荐 [bold]Qwen/Qwen3-Embedding-4B[/bold]，需 [bold yellow]1024 维[/bold yellow]且支持中英文的模型", - ), - "continue_hint": ("semantic recall will be unavailable", "语义召回将不可用"), - "skip_note": ( - " [yellow]! Skipped: recall will match keywords, not meaning.[/yellow]\n" - " [dim]Phrase a question differently and it may miss a memory you have.\n" - " Configure it later, then run `everos cascade backfill`.[/dim]", - " [yellow]⚠ 已跳过：召回将按关键词匹配，而非按语义。[/yellow]\n" - " [dim]换一种说法提问，就可能找不到已有的记忆。\n" - " 日后配好后运行 everos cascade backfill 可为已存记忆补上向量。[/dim]", - ), - }, - "rerank": { - "label": ("Memory rerank", "记忆 rerank"), - "example": "Qwen/Qwen3-Reranker-4B", - "optional": True, - "verify": True, - "purpose": ( - "Re-ranks what semantic search found so the best match comes first, at a small\n latency cost.", - "在语义召回一批候选后再精排一遍，让最相关的排在最前，会略增延迟。", - ), - "tag": ( - "[accent](optional, advised)[/accent]", - "[accent]（可选，建议配置）[/accent]", - ), - "recommendation": ( - "Recommended: [bold]Qwen/Qwen3-Reranker-4B[/bold]", - "推荐 [bold]Qwen/Qwen3-Reranker-4B[/bold]", - ), - "continue_hint": ("rerank quality may degrade", "rerank 精度可能下降"), - "skip_note": ( - " [dim]Skipped rerank; memory retrieval still works.[/dim]", - " [dim]已跳过 rerank，记忆检索仍可用。[/dim]", - ), - }, - "multimodal": { - "label": ("Memory multimodal", "记忆多模态"), - "example": "google/gemini-3-flash-preview", - "optional": True, - "verify": True, - "purpose": ( - "Lets Raven understand and recall images / PDFs / audio as memory.", - "让 Raven 把图片 / PDF / 音频也作为记忆来理解和检索。", - ), - "cost": ( - "Without it: those files stay out of memory. Having such files is not the same\n" - " as needing them remembered -- configure it when you do.", - "不配置：这类文件不进入记忆；有这类文件并不等于需要，确有此需求时再配即可。", - ), - "recommendation": ( - "Recommended: [bold]google/gemini-3-flash-preview[/bold]", - "推荐 [bold]google/gemini-3-flash-preview[/bold]", - ), - "skip_note": ( - " [dim]Skipped; nothing else is affected -- configure it if you come to need\n multimodal memory.[/dim]", - " [dim]已跳过；其余功能不受影响，日后确有把多模态内容纳入记忆的需求时再配即可。[/dim]", - ), - }, -} - - -_EMBEDDING_MODEL_PATTERNS = ("embed", "bge", "e5-", "gte-") -_MULTIMODAL_MODEL_PATTERNS = ("vision", "4o", "gemini", "pixtral", "qwen-vl", "qwen2-vl", "qwen2.5-vl") - - -def _fetch_everos_models( - base_url: Optional[str], - api_key: Optional[str], - *, - section: str = "llm", - provider_name: Optional[str] = None, -) -> Optional[list[str]]: - """Fetch available model ids from a provider endpoint. Never raises. - - For ``section="embedding"``, delegates to per-provider logic because - each provider exposes embedding models differently. - """ - if not base_url: - return None - if section == "embedding": - return _fetch_embedding_models(base_url, api_key, provider_name) - if section == "rerank": - return _fetch_rerank_models(base_url, api_key, provider_name) - if section == "multimodal": - return _fetch_multimodal_models(base_url, api_key, provider_name) - return _fetch_openai_models(base_url, api_key) - - -def _fetch_openai_models( - base_url: str, - api_key: Optional[str], - *, - params: Optional[dict[str, str]] = None, -) -> Optional[list[str]]: - """``GET {base_url}/models`` with OpenAI-style response parsing.""" - import httpx - - url = base_url.rstrip("/") + "/models" - headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} - try: - with httpx.Client(timeout=10) as client: - resp = client.get(url, headers=headers, params=params) - if resp.status_code != 200: - return None - data = resp.json() - except (httpx.HTTPError, httpx.InvalidURL, ValueError): - return None - items = data.get("data") if isinstance(data, dict) else None - if not isinstance(items, list): - return None - ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")] - return sorted(ids) or None - - -def _fetch_deepinfra_models( - api_key: Optional[str], - reported_type: str, - *, - name_contains: Optional[str] = None, -) -> Optional[list[str]]: - """Fetch DeepInfra models filtered by ``reported_type`` and optional name substring.""" - import httpx - - headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} - try: - with httpx.Client(timeout=10) as client: - resp = client.get("https://api.deepinfra.com/models/list", headers=headers) - if resp.status_code != 200: - return None - data = resp.json() - except (httpx.HTTPError, ValueError): - return None - items = data if isinstance(data, list) else [] - ids = [ - m.get("model_name") - for m in items - if isinstance(m, dict) - and m.get("reported_type") == reported_type - and m.get("model_name") - and (name_contains is None or name_contains in m.get("model_name", "")) - ] - return sorted(ids) or None - - -def _fetch_embedding_models( - base_url: str, - api_key: Optional[str], - provider_name: Optional[str], -) -> Optional[list[str]]: - """Provider-specific embedding model listing.""" - if provider_name == "openrouter": - return _fetch_openai_models(base_url.rstrip("/") + "/embeddings", api_key) - - if provider_name == "siliconflow": - return _fetch_openai_models(base_url, api_key, params={"type": "text", "sub_type": "embedding"}) - - if provider_name == "deepinfra": - return _fetch_deepinfra_models(api_key, "embeddings") - - # OpenAI, DashScope, custom — GET /models + name-based filter. - ids = _fetch_openai_models(base_url, api_key) - if ids is None: - return None - filtered = [i for i in ids if any(p in i.lower() for p in _EMBEDDING_MODEL_PATTERNS)] - return filtered or None - - -def _fetch_rerank_models( - base_url: str, - api_key: Optional[str], - provider_name: Optional[str], -) -> Optional[list[str]]: - """Provider-specific rerank model listing.""" - if provider_name == "deepinfra": - # The deepinfra provider hardcodes a Qwen3-Reranker chat template, - # so only Qwen3-Reranker models are compatible. - return _fetch_deepinfra_models(api_key, "reranker", name_contains="Qwen3-Reranker") - - if provider_name == "siliconflow": - return _fetch_openai_models(base_url, api_key, params={"sub_type": "reranker"}) - - if provider_name == "dashscope": - return ["gte-rerank-v2"] - - if provider_name == "openrouter": - return _fetch_openai_models(base_url, api_key, params={"output_modalities": "rerank"}) - - # vllm / custom — no standard rerank listing. - return None - - -def _fetch_multimodal_models( - base_url: str, - api_key: Optional[str], - provider_name: Optional[str], -) -> Optional[list[str]]: - """Provider-specific multimodal (vision) model listing.""" - if provider_name == "openrouter": - return _fetch_openai_models(base_url, api_key, params={"input_modalities": "image"}) - - # OpenAI, custom — GET /models + name-based filter. - ids = _fetch_openai_models(base_url, api_key) - if ids is None: - return None - filtered = [i for i in ids if any(p in i.lower() for p in _MULTIMODAL_MODEL_PATTERNS)] - return filtered or None - - -def _match_everos_default(example: str, models: list[str]) -> str: - """Find the best match for ``example`` in the fetched model list. - - The example (e.g. ``gpt-4.1-mini``) is a bare model name, while - ``models`` may carry provider prefixes (``openai/gpt-4.1-mini``). - Returns the first model whose id ends with ``/example`` or equals - ``example`` exactly; falls back to the bare example string so the - autocomplete input is pre-filled even if no exact match exists. - """ - lower = example.lower() - suffix = f"/{lower}" - for mid in models: - if mid.lower() == lower or mid.lower().endswith(suffix): - return mid - return example - - -def _preferred_memory_model(section: str, main_model: Optional[str], chosen_provider: Optional[str]) -> Optional[str]: - """The main chat model, when it is a sensible pre-fill for this role. - - Only the llm role -- an embedding / rerank / multimodal endpoint does not - serve a chat model. Only when the picked provider is the main model's own: no - other provider carries that id, and pre-filling one it cannot serve turns - Enter into a verification failure. A custom endpoint has no resolved provider - and is left alone for the same reason. - """ - if section != "llm" or not main_model or chosen_provider is None: - return None - if chosen_provider != _resolve_model_provider(main_model): - return None - return _resolve_reuse_llm_creds(main_model).get("model") - - -def _everos_pick_model( - *, - base_url: Optional[str], - api_key: Optional[str], - example: str, - allow_back: bool, - section: str = "llm", - provider_name: Optional[str] = None, - recommendation: Optional[tuple[str, str]] = None, - preferred: Optional[str] = None, -) -> Any: - """Pick a model id for an EverOS endpoint: fetch ``/models`` for a - fuzzy-searchable list, else fall back to free text. Empty submit = back. - - ``preferred`` pre-fills a model the user is already known to have access to - -- their main chat model. It wins over ``example`` because a recommended - model is only a recommendation if the user's key can reach it, and many keys - cannot; ``example`` then reads as the capability floor rather than the - default (see ``recommendation``). - """ - questionary = _require_questionary() - from raven.cli._styles import RAVEN_STYLE - - console.print(_t(" [dim]⏳ Loading models…[/dim]", " [dim]⏳ 正在拉取模型列表…[/dim]")) - models = _fetch_everos_models(base_url, api_key, section=section, provider_name=provider_name) - if preferred: - console.print( - _t( - f" [dim]Pre-filled with your main model [bold]{preferred}[/bold] -- press Enter to accept.[/dim]", - f" [dim]已填入你的主模型 [bold]{preferred}[/bold]，直接回车即可。[/dim]", - ) - ) - if recommendation: - console.print(f" [dim]{_t(*recommendation)}[/dim]") - if models: - default_model = preferred or _match_everos_default(example, models) - question = questionary.autocomplete( - _t( - f"Model ({len(models)} available — type to filter):", - f"模型(共 {len(models)} 个 — 输入可筛选):", - ), - choices=models, - default=default_model, - ignore_case=True, - match_middle=True, - placeholder=_back_placeholder(allow_back), - style=RAVEN_STYLE, - qmark=_QMARK, - ) - # Trigger the completion popup immediately so the user sees - # all available models without typing first. - app = question.application - - def _show_completions() -> None: - buf = app.current_buffer - buf.start_completion() - - app.pre_run_callables.append(_show_completions) - chosen = question.ask() - else: - console.print( - _t( - " [dim]Couldn't list models from this endpoint — type the id manually.[/dim]", - " [dim]该端点拉不到模型列表 — 请手动输入模型 id。[/dim]", - ) - ) - chosen = questionary.text( - _t(f"Model id (e.g. {example}):", f"模型 id（如 {example}）："), - default=preferred or "", - placeholder=_back_placeholder(allow_back), - style=RAVEN_STYLE, - qmark=_QMARK, - ).ask() - if chosen is None: - raise typer.Exit(1) - chosen = chosen.strip() - if allow_back and chosen == "": - return _BACK - if not chosen: - raise typer.Exit(1) - return chosen - - -def _everos_pick_creds_and_model( - *, - section: str, - example: str, - main_model: Optional[str], - non_interactive: bool, - recommendation: Optional[tuple[str, str]] = None, -) -> Any: - """Mirror the main provider step for one EverOS model: pick a source - (curated provider / custom) → API key → model. Returns a dict with - ``model`` / ``api_key`` / ``base_url`` (plus ``provider`` for rerank), or - ``_BACK`` when the user backs out of the source picker. Empty submit on any - field rewinds one step.""" - questionary = _require_questionary() - from raven.cli._styles import RAVEN_STYLE - - llm_section = _everos_section("llm") - - # For the LLM role, default to the main chat model's provider. - # For other roles (embedding/rerank/multimodal), default to whichever - # provider the LLM step just configured — the user likely has the - # same API key and only needs to pick a different model. - if section == "llm": - default_provider = _resolve_model_provider(main_model or "") - reuse_source = "main" - else: - default_provider = _match_provider_by_url(llm_section.get("base_url")) - reuse_source = "llm" - - while True: # source picker — a field-level back rewinds here - choices: list[Any] = [] - default_choice = None - for prov in _EVEROS_PROVIDERS: - if section not in prov.get("supports", set()): - continue - is_default = default_provider is not None and prov["name"] == default_provider - if is_default: - if reuse_source == "main": - label = _t( - f"{prov['label']} (main model provider, reuse Key)", - f"{prov['label_zh']}（主模型服务商，复用 Key）", - ) - else: - label = _t( - f"{prov['label']} (memory LLM provider, reuse Key)", - f"{prov['label_zh']}（记忆 LLM 服务商，复用 Key）", - ) - else: - label = _t(prov["label"], prov["label_zh"]) - choice = questionary.Choice(label, value=("provider", prov)) - choices.append(choice) - if is_default: - default_choice = choice.value - choices.append( - questionary.Choice( - _t("Other (custom OpenAI-compatible endpoint)", "其他(自定义 OpenAI 兼容端点)"), - value=("custom",), - ) - ) - choices.append(questionary.Separator()) - choices.append(questionary.Choice(_t("Back", "返回"), value=_BACK)) - - src = questionary.select( - _t("Pick a provider (or reuse / custom):", "选择服务商(或复用 / 自定义):"), - choices=choices, - default=default_choice, - style=RAVEN_STYLE, - qmark=_QMARK, - ).ask() - if src is None: - raise typer.Exit(1) - if src is _BACK: - return _BACK - kind = src[0] - - # Resolve (api_key, base_url) from the chosen source. - chosen_provider: Optional[str] = None - if kind == "provider": - chosen_provider = src[1]["name"] - base_url = src[1]["base_url"] - prefilled_key: Optional[str] = None - if default_provider == src[1]["name"]: - if reuse_source == "main": - prefilled_key = _resolve_reuse_llm_creds(main_model or "").get("api_key") - else: - prefilled_key = llm_section.get("api_key") - if prefilled_key: - if reuse_source == "main": - console.print( - _t( - " [dim]API key reused from main chat model.[/dim]", - " [dim]已复用主对话模型的 API Key。[/dim]", - ) - ) - else: - console.print( - _t( - " [dim]API key reused from memory LLM.[/dim]", - " [dim]已复用记忆 LLM 的 API Key。[/dim]", - ) - ) - api_key = prefilled_key - else: - api_key = _prompt_api_key(src[1]["name"], allow_back=True) - if api_key is _BACK: - continue - else: # custom - base_url = _prompt_text(_t("Base URL (must include /v1):", "Base URL(需包含 /v1):"), allow_back=True) - if base_url is _BACK: - continue - api_key = _prompt_text(_t("API key (hidden):", "API Key(隐藏输入):"), secret=True, allow_back=True) - if api_key is _BACK: - continue - - # Guard against a source that resolved to an empty key / endpoint — - # set_everos_section drops None values, which would otherwise persist a - # section with a model but no usable endpoint. - if not (api_key and base_url): - console.print( - _t( - " [yellow]✗ Missing API key or Base URL for this source — pick another.[/yellow]", - " [yellow]✗ 该来源缺少 API Key 或 Base URL — 请换一个。[/yellow]", - ) - ) - continue - - # rerank: resolve service type + override base_url when needed. - rerank_provider: Optional[str] = None - if section == "rerank": - chosen_prov_dict = src[1] if kind == "provider" else None - if chosen_prov_dict and chosen_prov_dict.get("rerank_provider"): - rerank_provider = chosen_prov_dict["rerank_provider"] - if chosen_prov_dict.get("rerank_base_url"): - base_url = chosen_prov_dict["rerank_base_url"] - else: - rerank_provider = questionary.select( - _t("Rerank service type:", "rerank 服务类型:"), - choices=[ - questionary.Choice("deepinfra", value="deepinfra"), - questionary.Choice("vllm", value="vllm"), - questionary.Choice("dashscope", value="dashscope"), - questionary.Choice(_t("Back", "返回"), value=_BACK), - ], - style=RAVEN_STYLE, - qmark=_QMARK, - ).ask() - if rerank_provider is None: - raise typer.Exit(1) - if rerank_provider is _BACK: - continue - - model = _everos_pick_model( - base_url=base_url, - api_key=api_key, - example=example, - allow_back=True, - section=section, - provider_name=chosen_provider, - recommendation=recommendation, - preferred=_preferred_memory_model(section, main_model, chosen_provider), - ) - if model is _BACK: - continue - - result: dict[str, Any] = {"model": model, "api_key": api_key, "base_url": base_url} - if rerank_provider: - result["provider"] = rerank_provider - return result - - -def _config_everos_role( - *, section: str, main_model: Optional[str], non_interactive: bool, warnings: list[str], skip_test: bool = False -) -> Any: - """Configure one EverOS memory role (llm / embedding / rerank / multimodal) - with the unified provider→key→model flow, reuse shortcuts, and a back loop. - - Returns ``None`` normally; returns ``_ABORT_EVEROS`` when the user gives up a - required role (the caller then disables EverOS, leaving no long-term memory).""" - questionary = _require_questionary() - from raven.cli._styles import RAVEN_STYLE - from raven.config.update_everos import clear_everos_section, set_everos_section - - role = _EVEROS_ROLES[section] - label_en, label_zh = role["label"] - purpose_en, purpose_zh = role["purpose"] - optional = role["optional"] - verify_label = _t(label_en, label_zh) - - # Tell the user what this model is for, and what skipping it costs, before - # asking them to configure it. Header sits on the 2-space info column (bold - # accent); purpose and cost nest under it, matching the layout used - # everywhere else. - # - # The cost line is dim rather than a warning colour on purpose: this is - # pre-decision information, and colouring it would cry wolf before the user - # has chosen anything. The warning comes after, from ``skip_note``. - # - # Roles that want to be configured say so in their own ``tag`` -- calling all - # three merely "optional" flattens the difference between losing semantic - # recall entirely and losing a little ranking accuracy. - tag_markup = _t(*role["tag"]) if role.get("tag") else _t("[dim](optional)[/dim]", "[dim]（可选）[/dim]") - lines = [f" [bold][accent]{_t(label_en, label_zh)}[/accent][/bold]" + (f" {tag_markup}" if optional else "")] - lines.append(f" [dim]{_t(purpose_en, purpose_zh)}[/dim]") - if role.get("cost"): - lines.append(f" [dim]{_t(*role['cost'])}[/dim]") - console.print() - # highlight=False so Rich's default highlighter doesn't tint the dim prose - # (parens/numbers/words) and make an informational hint read like an error. - console.print("\n".join(lines), highlight=False) - - while True: # role-menu loop — a back-out of the source picker returns here - current = _everos_section(section).get("model") if _everos_role_configured(section) else None - if current: - choices = [ - questionary.Choice(_t(f"Keep current: {current}", f"沿用当前:{current}"), value="keep"), - questionary.Choice(_t("Reconfigure", "重新配置"), value="redo"), - ] - if optional: - choices.append(questionary.Choice(_t("Skip", "跳过"), value="off")) - action = questionary.select( - _t("Already configured — what now?", "已配置,怎么处理?"), - choices=choices, - style=RAVEN_STYLE, - qmark=_QMARK, - ).ask() - if action is None: - raise typer.Exit(1) - if action == "keep": - return - if action == "off": - clear_everos_section(section) - console.print(_t(f" [dim]{label_en} skipped.[/dim]", f" [dim]已跳过 {label_zh}。[/dim]")) - return - elif optional: - action = questionary.select( - _t("Configure it?", "要配置吗?"), - choices=[ - questionary.Choice(_t("Configure", "配置"), value="redo"), - questionary.Choice(_t("Skip", "跳过"), value="skip"), - ], - style=RAVEN_STYLE, - qmark=_QMARK, - ).ask() - if action is None: - raise typer.Exit(1) - if action == "skip": - # Printed verbatim rather than wrapped in [dim]: skipping rerank - # costs ordering, skipping embedding costs semantic recall - # entirely, and one of those deserves to be seen. - note_en, note_zh = role.get( - "skip_note", (f" [dim]Skipped {label_en}.[/dim]", f" [dim]已跳过 {label_zh}。[/dim]") - ) - console.print(_t(note_en, note_zh), highlight=False) - return - # A required role with nothing configured falls straight into the picker. - - result = _everos_pick_creds_and_model( - section=section, - example=role["example"], - main_model=main_model, - non_interactive=non_interactive, - recommendation=role.get("recommendation"), - ) - if result is _BACK: - if optional or _everos_role_configured(section): - # Optional roles offer Skip; a required role already configured - # falls back to its keep/reconfigure menu. Either way, re-show - # the role menu rather than forcing the give-up exit. - continue - # A required role with nothing configured has no Skip, so backing out - # of the picker would loop forever. Offer a bounded exit -- keep - # trying, or leave without long-term memory. Stated in full and in - # colour: this is the only place the wizard can lose memory - # altogether, and "no cross-session memory" is a consequence a user - # should not discover weeks later by noticing the agent forgets - # everything. - console.print() - console.print( - _t( - f" [yellow]⚠ {label_en} is required for long-term memory.[/yellow]\n" - " [dim]Without it Raven has no memory across sessions: every conversation starts\n" - " from nothing, with no recollection of your preferences or of what was done before.[/dim]", - f" [yellow]⚠ {label_zh} 是长期记忆的必需项。[/yellow]\n" - " [dim]放弃后 Raven 没有任何跨会话记忆：每次对话都从零开始，不记得你的偏好，\n" - " 也不记得之前做过什么。[/dim]", - ), - highlight=False, - ) - action = questionary.select( - _t("What would you like to do?", "想做什么？"), - choices=[ - questionary.Choice(_t("Pick a provider / model", "选择服务商 / 模型"), value="retry"), - questionary.Choice( - _t("Give up (no long-term memory)", "放弃（不启用长期记忆）"), - value="abort", - ), - ], - style=RAVEN_STYLE, - qmark=_QMARK, - ).ask() - if action is None: - raise typer.Exit(1) - if action == "retry": - continue - return _ABORT_EVEROS - - if role["verify"] and skip_test: - console.print( - _t( - f" [dim]Skipping the {verify_label} test call (--skip-test).[/dim]", - f" [dim]已跳过 {verify_label} 的测试调用(--skip-test)。[/dim]", - ) - ) - ok = True - elif section == "llm": - ok = _verify_everos_llm( - verify_label, - model=result["model"], - api_key=result["api_key"], - base_url=result["base_url"], - non_interactive=non_interactive, - warnings=warnings, - continue_hint=role.get("continue_hint"), - ) - elif section == "embedding": - ok = _verify_embedding_dim( - model=result["model"], - api_key=result["api_key"], - base_url=result["base_url"], - non_interactive=non_interactive, - ) - elif section == "rerank": - ok = _verify_rerank( - verify_label, - model=result["model"], - api_key=result["api_key"], - base_url=result["base_url"], - rerank_provider=result.get("provider"), - non_interactive=non_interactive, - warnings=warnings, - continue_hint=role.get("continue_hint"), - ) - elif section == "multimodal": - ok = _verify_everos_llm( - verify_label, - model=result["model"], - api_key=result["api_key"], - base_url=result["base_url"], - non_interactive=non_interactive, - warnings=warnings, - continue_hint=role.get("continue_hint"), - ) - else: - ok = True - if not ok: - continue - - set_everos_section(section, result) - console.print( - _t( - f" [green]✓ {label_en} configured.[/green]", - f" [green]✓ 已配置 {label_zh}。[/green]", - ) - ) - return - - -def _step4_memory( - *, skip: bool, non_interactive: bool, main_model: Optional[str], warnings: list[str], skip_test: bool = False -) -> object: - """Step 4 -- EverOS long-term memory (model sub-screens). - - The bootstrap seeds ``memory.backend="everos"`` (schema default) and everos - is the only memory backend, so this step does not ask whether to enable it: - it either confirms the seed by configuring the llm role, or resolves it back - to ``None`` on skip / non-interactive / give-up. ``None`` means no long-term - memory at all, not a fallback to something simpler. - - ``_memory_enabled`` gates on the llm role alone, so a fresh modelless seed - reads as "not configured yet" and the keep/reconfigure menu only appears once - that model is actually on disk. embedding and rerank are offered here but - never gate: skipping them costs recall quality, not memory itself. - """ - _step_header(4, _t("EverOS long-term memory", "EverOS 长期记忆")) - - import sys - - if sys.platform == "win32": - console.print( - _t( - " [yellow]⚠ EverOS memory engine does not support native Windows.[/yellow]\n" - " [dim]Run Raven inside WSL for full memory support.[/dim]\n" - " [dim]Skipping memory configuration.[/dim]", - " [yellow]⚠ EverOS 记忆引擎暂不支持 Windows 原生环境。[/yellow]\n" - " [dim]在 WSL 中运行 Raven 可获得完整记忆支持。[/dim]\n" - " [dim]已跳过记忆配置。[/dim]", - ) - ) - _set_memory_backend(None) - return None - - if skip or non_interactive: - # Never configured the required models here → disable backend-driven - # memory so runtime doesn't activate EverOS without an llm/embedding. - # (``_memory_enabled`` already gates on both required models, so an - # already-enabled+configured setup is preserved.) - if not _memory_enabled(): - _set_memory_backend(None) - console.print( - _t( - " [dim]Long-term memory stays off.[/dim]", - " [dim]长期记忆保持关闭。[/dim]", - ) - ) - return None - - questionary = _require_questionary() - from raven.cli._styles import RAVEN_STYLE - - if _memory_enabled(): - action = questionary.select( - _t( - "EverOS long-term memory is already enabled. What would you like to do?", - "EverOS 长期记忆已启用。想做什么?", - ), - choices=[ - questionary.Choice(_t("Keep it enabled", "保持启用"), value="keep"), - questionary.Choice(_t("Reconfigure", "重新配置"), value="redo"), - ], - style=RAVEN_STYLE, - qmark=_QMARK, - ).ask() - if action is None: - raise typer.Exit(1) - if action == "keep": - return None # backend already "everos" + models on disk; leave as-is - else: - # No enable/decline question: everos is the only memory backend, so the - # step goes straight into configuring it. Leaving is still possible -- - # backing out of the required roles reaches the give-up prompt, which - # spells out what is lost. - # Wrapped by hand: rich re-wraps at the terminal width and drops the - # two-space indent on continuation lines, which reads as a stray - # left-flush sentence under an indented block. - console.print( - _t( - " [dim]Raven's long-term memory comes from EverOS. What it can do grows with\n" - " what you configure:[/dim]\n" - " [dim] memory LLM only conversations become memories; recall matches keywords[/dim]\n" - " [dim] + memory embedding recall matches meaning, not wording (strongly advised)[/dim]\n" - " [dim] + memory rerank recall ordering gets sharper[/dim]", - " [dim]Raven 拥有 EverOS 提供的强大长期记忆能力，能力随配置递进：[/dim]\n" - " [dim] 仅记忆 LLM 对话会被提炼成记忆存下来，召回按关键词匹配[/dim]\n" - " [dim] + 记忆 embedding 召回按语义匹配，换个问法也能找到（强烈建议配）[/dim]\n" - " [dim] + 记忆 rerank 召回结果排序更准[/dim]", - ), - highlight=False, - ) - - # Ensure the EverOS home directory has its config templates (everos.toml - # + ome.toml) BEFORE writing model sections — set_everos_section merges - # into the template so default sections (memory/sqlite/lancedb/api) are - # preserved. Also creates ome.toml which the runtime requires. - from raven.config.update_everos import configure_everos_env, ensure_everos_home - - configure_everos_env() - ensure_everos_home() - - # Configure required models FIRST, then flip the backend on — so a Ctrl+C - # mid-configuration leaves backend at its prior (disabled) value rather - # than an enabled-but-modelless state. - for _role in ("llm", "embedding", "rerank", "multimodal"): - # Each role prints one leading blank before its own header, so no extra - # separator here — avoids the double blank line between roles. - outcome = _config_everos_role( - section=_role, - main_model=main_model, - non_interactive=non_interactive, - warnings=warnings, - skip_test=skip_test, - ) - if outcome is _ABORT_EVEROS: - _set_memory_backend(None) - console.print( - _t( - " [yellow]⚠ Gave up long-term memory: Raven will not remember anything " - "between sessions.[/yellow]\n" - " [dim]Run `raven onboard` again whenever you want to configure it.[/dim]", - " [yellow]⚠ 已放弃长期记忆，Raven 不会记住任何跨会话内容。[/yellow]\n" - " [dim]随时可以重新运行 raven onboard 配置。[/dim]", - ) - ) - return None - - # Verify EverOS server is reachable (auto-starts if needed) - import asyncio - - from raven.plugin.memory.everos._server import ensure_everos_server - - console.print() - console.print( - _t( - " [dim]Starting EverOS service...[/dim]", - " [dim]正在启动 EverOS 服务...[/dim]", - ) - ) - try: - asyncio.run(ensure_everos_server()) - console.print( - _t( - " [green]✓ EverOS service is running.[/green]", - " [green]✓ EverOS 服务已启动。[/green]", - ) - ) - except RuntimeError as exc: - console.print( - _t( - f" [red]✗ EverOS service failed to start: {exc}[/red]\n" - " [dim]Check: everos installed? Port 18791 free? " - "See ~/.raven/logs/everos-server.log[/dim]", - f" [red]✗ EverOS 服务启动失败：{exc}[/red]\n" - " [dim]请检查：everos 是否安装？端口 18791 是否被占用？" - "查看 ~/.raven/logs/everos-server.log[/dim]", - ) - ) - retry = questionary.select( - _t("What to do?", "怎么办？"), - choices=[ - questionary.Choice(_t("Retry", "重试"), value="retry"), - questionary.Choice(_t("Skip (memory disabled)", "跳过（记忆禁用）"), value="skip"), - ], - style=RAVEN_STYLE, - qmark=_QMARK, - ).ask() - if retry == "retry": - # Recurse once — the loop in _step4_memory handles further retries - try: - asyncio.run(ensure_everos_server()) - console.print( - _t( - " [green]✓ EverOS service is running.[/green]", - " [green]✓ EverOS 服务已启动。[/green]", - ) - ) - except RuntimeError: - console.print( - _t( - " [red]✗ Still failed. Disabling memory.[/red]", - " [red]✗ 仍然失败。禁用记忆功能。[/red]", - ) - ) - _set_memory_backend(None) - return None - else: - _set_memory_backend(None) - return None - _report_everos_capabilities() - _set_memory_backend("everos") - return None - - -def _report_everos_capabilities() -> None: - """Say what the running server can actually do, not just that it answers. - - ``ensure_everos_server`` proves the process is up and nothing more. Since - everos 1.2.1 a server whose embedding provider failed to build still answers - 200 and degrades to keyword-only search, so stopping at "running" would - print a tick over an install that cannot recall anything. The roles were - each verified against their provider earlier in this step; what is new here - is whether everos itself could build them from what got written to - ``everos.toml``. - - Silent on a server too old to report capabilities -- reading that silence as - "unavailable" would condemn a working install. - """ - from raven.config.raven import load_raven_config - from raven.plugin.memory.everos._health import ( - DEGRADING_SECTIONS, - REQUIRED_SECTIONS, - configured_base_url, - probe_capabilities, - ) - - # The configured address, not the default: probing the wrong port reports on - # a server nobody is using, and reads as "not running". - report = probe_capabilities(configured_base_url(load_raven_config())) - if not report.reports_capabilities: - return - configured = [s for s in (*REQUIRED_SECTIONS, *DEGRADING_SECTIONS) if _everos_role_configured(s)] - broken = [s for s in configured if report.available(s) is False] - if not broken: - names = " and ".join(configured) - console.print( - _t( - f" [green]✓ {names} {'is' if len(configured) == 1 else 'are'} available.[/green]", - f" [green]✓ {names} 均可用。[/green]", - ) - ) - return - names = " and ".join(broken) - console.print( - _t( - f" [yellow]⚠ {names} is configured but EverOS could not build it.[/yellow]\n" - " [dim]Memory runs degraded until this is fixed.[/dim]\n" - f" [dim]Check: {_everos_server_log_hint()}[/dim]", - f" [yellow]⚠ {names} 已配置，但 EverOS 未能构建成功。[/yellow]\n" - " [dim]在此修复前，记忆能力将处于降级状态。[/dim]\n" - f" [dim]请查看：{_everos_server_log_hint()}[/dim]", - ) - ) - - -def _everos_server_log_hint() -> str: - from raven.plugin.memory.everos._server import server_log_path - - return str(server_log_path()) - - # --------------------------------------------------------------------------- # Final summary # --------------------------------------------------------------------------- @@ -4252,7 +2743,11 @@ def _print_next_steps(*, warnings: list[str]) -> None: else _t("Sandbox (boxlite)", "沙箱(boxlite)") ) chans = ", ".join(_enabled_channels()) or _t("none", "无") - mem = _t("EverOS", "EverOS") if _memory_enabled() else _t("[yellow]off[/yellow]", "[yellow]未启用[/yellow]") + mem = ( + _t("EverOS", "EverOS") + if onboard_everos._memory_enabled() + else _t("[yellow]off[/yellow]", "[yellow]未启用[/yellow]") + ) recap = Table(show_header=False, box=None, padding=(0, 2, 0, 0)) recap.add_column(style="dim", no_wrap=True) recap.add_column() @@ -4338,7 +2833,7 @@ def _step5_import(*, skip: bool, non_interactive: bool) -> object: ) return None - if not _memory_enabled(): + if not onboard_everos._memory_enabled(): console.print( _t( " [dim]Skipped — EverOS long-term memory is required for history import.[/dim]", @@ -4923,7 +3418,7 @@ def _run_wizard_body( ), lambda: _step2_sandbox(skip=skip_sandbox, non_interactive=non_interactive), lambda: _step3_channel(channel=channel, skip=skip_channel, non_interactive=non_interactive), - lambda: _step4_memory( + lambda: onboard_everos._step4_memory( skip=skip_memory, non_interactive=non_interactive, main_model=_load_current_default_model(), diff --git a/raven/cli/onboard_everos.py b/raven/cli/onboard_everos.py new file mode 100644 index 00000000..13fa6607 --- /dev/null +++ b/raven/cli/onboard_everos.py @@ -0,0 +1,1525 @@ +"""EverOS long-term memory cluster of the onboard wizard (Step 4). + +Split out of ``onboard_commands`` because that module had grown past 5000 +lines; this file owns EverOS role configuration (llm / embedding / rerank / +multimodal) end to end. Shared wizard UI state (``console``, ``_t``, ``_BACK``, +``_QMARK``, questionary helpers, ...) still lives in ``onboard_commands`` -- +this module reaches it via the ``oc`` module reference (not a value import) so +that test monkeypatches on ``onboard_commands`` attributes keep working +whichever module a caller patches through. +""" + +from __future__ import annotations + +from typing import Any, Optional + +import typer + +from raven.cli import onboard_commands as oc + + +def _set_memory_backend(backend: Optional[str]) -> None: + """Set ``memory.backend`` (``"everos"`` / ``None``) via the ops layer.""" + from raven.config.update import set_memory_backend + + set_memory_backend(backend) + + +def _init_extension_block_defaults() -> None: + """Seed the memory / plugins / skillForge extension defaults via the ops layer.""" + from raven.config.update import init_extension_block_defaults + + init_extension_block_defaults() + + +def _everos_section(section: str) -> dict[str, Any]: + from raven.config.update_everos import everos_section + + return everos_section(section) + + +def _everos_role_configured(section: str) -> bool: + from raven.config.update_everos import everos_role_configured + + return everos_role_configured(section) + + +def _memory_enabled() -> bool: + """True iff EverOS memory is both selected AND usable on disk. + + "Usable" means the llm role is configured -- that is the whole requirement. + embedding is advised but optional: without it the adapter searches lexically + instead of semantically, which is weaker memory rather than none, and gating + on it here would tell a user who skipped it that memory is off (and skip the + import step along with it). + """ + data = oc._load_raw_config() + if (data.get("memory") or {}).get("backend") != "everos": + return False + return _everos_role_configured("llm") + + +# Providers whose main model can be reused as the EverOS memory LLM: they +# speak the OpenAI chat-completions protocol that EverOS's bare OpenAI client +# requires. OAuth providers (github_copilot / openai_codex) and non-OpenAI +# wire protocols (anthropic / gemini) are excluded. +_OPENAI_COMPATIBLE_PROVIDERS = {"openrouter", "openai", "deepseek", "custom"} + +# Fallback OpenAI-compatible base URLs for providers whose registry +# ``default_api_base`` is empty (they rely on the SDK's built-in default, +# which EverOS's bare client doesn't know). EverOS needs an explicit base_url. +_PROVIDER_BASE_URL_FALLBACK = { + "openai": "https://api.openai.com/v1", + "deepseek": "https://api.deepseek.com/v1", + "openrouter": "https://openrouter.ai/api/v1", +} + + +def _resolve_model_provider(model: str) -> Optional[str]: + """Best-effort: which configured provider does ``model`` belong to? + + Prefixed models (``openrouter/...`` / ``openai/gpt-4o``) read off the head. + A custom endpoint stores its model as a BARE id (e.g. ``qwen-max``) with no + prefix, so an unrecognized head falls back to ``"custom"`` when a custom + provider is actually configured with a key. Returns ``None`` when no match. + """ + from raven.providers.registry import split_model_id + + if not model: + return None + head, _ = split_model_id(model) + if head: + from raven.config.update_providers import provider_field_specs + + try: + provider_field_specs(head) + return head + except KeyError: + pass + # No usable prefix → could be a bare custom-endpoint model. + from raven.config.schema import ProviderConfig + from raven.providers.auth import credential_status + + custom = (oc._load_raw_config().get("providers") or {}).get("custom") or {} + if credential_status("custom", ProviderConfig.model_validate(custom)).ok: + return "custom" + # A bare id that still matches a known provider head (rare; e.g. a direct + # provider's bare default before prefixing) — accept the head if known. + return head if head in _OPENAI_COMPATIBLE_PROVIDERS else None + + +def _model_is_openai_compatible(model: Optional[str]) -> bool: + """Heuristic: can the main chat model's provider be reused for memory LLM? + + EverOS's memory LLM uses a bare OpenAI client, so the main model is + reusable only when its provider speaks the OpenAI chat protocol. Custom + endpoints are OpenAI-compatible by definition (the wizard only offers + ``custom`` for OpenAI-compatible endpoints). + """ + if not model: + return False + return _resolve_model_provider(model) in _OPENAI_COMPATIBLE_PROVIDERS + + +def _resolve_reuse_llm_creds(main_model: str) -> dict[str, Optional[str]]: + """Map a litellm-style main model to bare EverOS LLM settings. + + EverOS sends ``EVEROS_LLM__MODEL`` to ``base_url`` via a bare OpenAI + client, so: + - strip the provider's litellm prefix to the bare model id the upstream + endpoint expects (``openrouter/anthropic/claude-x`` → ``anthropic/claude-x``; + a custom endpoint's bare id is used as-is); + - resolve the provider's real ``base_url`` (configured ``apiBase`` → + registry ``default_api_base`` → a known fallback); + - carry the provider's stored api_key. + """ + from raven.providers.registry import find_by_name, normalize_provider_name, split_model_id + + provider = _resolve_model_provider(main_model) or split_model_id(main_model)[0] + spec = find_by_name(provider) + # Through the ops library, so a section still stored under the provider's + # pre-rename name is found -- a raw lookup by the resolved name is not. + from raven.config.update_providers import get_provider_config + + # No `if spec` gate: LiteLLM-only vendors have no spec of ours yet their + # section holds real credentials, and gating on the spec silently handed the + # probe an empty api_key while the main model was working fine. + try: + _resolved = get_provider_config(provider, redact_secrets=False) + except KeyError: + _resolved = {} + prov_cfg = {"apiKey": _resolved.get("api_key"), "apiBase": _resolved.get("api_base")} if _resolved else {} + + # Strip the routing prefix to the bare model id the upstream endpoint + # expects: litellm consumes it, the raw OpenAI client must not see it. Only + # a prefix naming this provider is stripped -- a custom endpoint stores a + # bare id already, and anything else is part of the vendor's own model id. + bare_model = main_model + head, rest = split_model_id(main_model) + known_prefixes = set(spec.route_names) if spec else {normalize_provider_name(provider)} + if spec: + known_prefixes.add(normalize_provider_name(spec.model_prefix)) + if head and head in known_prefixes: + bare_model = rest + + base_url = ( + prov_cfg.get("apiBase") + or (getattr(spec, "default_api_base", "") if spec else "") + or _PROVIDER_BASE_URL_FALLBACK.get(provider) + ) + return { + "model": bare_model, + "api_key": prov_cfg.get("apiKey"), + "base_url": base_url, + } + + +def _prompt_text(label: str, *, secret: bool = False, default: str = "", allow_back: bool = False) -> Any: + """Prompt for free text. With ``allow_back``, an empty submit returns + ``oc._BACK`` (and a hint is shown); otherwise returns the stripped string.""" + questionary = oc._require_questionary() + from raven.cli._styles import RAVEN_STYLE + + placeholder = oc._back_placeholder(allow_back) + if secret: + value = questionary.password(label, placeholder=placeholder, style=RAVEN_STYLE, qmark=oc._QMARK).ask() + else: + value = questionary.text( + label, default=default, placeholder=placeholder, style=RAVEN_STYLE, qmark=oc._QMARK + ).ask() + if value is None: + raise typer.Exit(1) + value = value.strip() + if allow_back and value == "": + return oc._BACK + return value + + +def _probe_everos_chat(model: Optional[str], *, api_key: Optional[str], base_url: Optional[str]) -> tuple[bool, str]: + """Real capability probe for a memory-LLM endpoint: ``POST + {base_url}/chat/completions`` once and confirm a choice comes back. Unlike a + bare ``GET /models`` connectivity check, this exercises the picked model, so + an endpoint that lists models but doesn't serve the chosen id fails here + instead of reporting a false green. Provider-agnostic; never raises.""" + import httpx + + if not base_url: + return False, "no base_url configured" + url = base_url.rstrip("/") + ("/chat/completions" if "/v1" in base_url else "/v1/chat/completions") + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + body = {"model": model, "messages": [{"role": "user", "content": "ping"}], "max_tokens": 1} + try: + with httpx.Client(timeout=15) as client: + resp = client.post(url, headers=headers, json=body) + if resp.status_code != 200: + return False, f"HTTP {resp.status_code}: {resp.text[:200]}" + data = resp.json() + except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc: + return False, f"probe failed: {exc}" + choices = data.get("choices") if isinstance(data, dict) else None + if isinstance(choices, list) and choices and isinstance(choices[0], dict): + return True, "ok" + return False, "endpoint returned no completion" + + +def _verify_everos_llm( + label: str, + *, + model: Optional[str], + api_key: Optional[str], + base_url: Optional[str], + non_interactive: bool, + warnings: list[str], + continue_hint: Optional[tuple[str, str]] = None, +) -> bool: + """Probe the memory LLM with a real chat completion, offering retry/continue on failure.""" + oc.console.print(oc._t(f" [dim]⏳ Verifying {label}…[/dim]", f" [dim]⏳ 正在验证 {label}…[/dim]")) + ok, detail = _probe_everos_chat(model, api_key=api_key, base_url=base_url) + if ok: + oc.console.print(oc._t(f" [green]✓ {label} connected.[/green]", f" [green]✓ {label} 连接成功。[/green]")) + return True + oc.console.print( + oc._t( + f" [yellow]✗ Couldn't verify {label}: {detail}[/yellow]", + f" [yellow]✗ 验证失败 {label}:{detail}[/yellow]", + ) + ) + if continue_hint: + cont_label = oc._t(f"Continue anyway ({continue_hint[0]})", f"仍然继续({continue_hint[1]})") + else: + cont_label = oc._t("Continue anyway", "仍然继续") + choice = oc._failure_choice( + [ + (oc._t("Re-enter", "重新填写"), "rekey"), + (cont_label, "continue"), + ], + non_interactive=non_interactive, + ) + if choice == "rekey": + return False + warnings.append(label) + return True + + +def _verify_rerank( + label: str, + *, + model: Optional[str], + api_key: Optional[str], + base_url: Optional[str], + rerank_provider: Optional[str], + non_interactive: bool, + warnings: list[str], + continue_hint: Optional[tuple[str, str]] = None, +) -> bool: + """Probe a rerank endpoint with a provider-specific request, offering retry/continue on failure.""" + oc.console.print(oc._t(f" [dim]⏳ Verifying {label}…[/dim]", f" [dim]⏳ 正在验证 {label}…[/dim]")) + ok, detail = _probe_rerank(model, api_key=api_key, base_url=base_url, rerank_provider=rerank_provider) + if ok: + oc.console.print(oc._t(f" [green]✓ {label} connected.[/green]", f" [green]✓ {label} 连接成功。[/green]")) + return True + oc.console.print( + oc._t( + f" [yellow]✗ Couldn't verify {label}: {detail}[/yellow]", + f" [yellow]✗ 验证失败 {label}:{detail}[/yellow]", + ) + ) + if continue_hint: + cont_label = oc._t(f"Continue anyway ({continue_hint[0]})", f"仍然继续({continue_hint[1]})") + else: + cont_label = oc._t("Continue anyway", "仍然继续") + choice = oc._failure_choice( + [ + (oc._t("Re-enter", "重新填写"), "rekey"), + (cont_label, "continue"), + ], + non_interactive=non_interactive, + ) + if choice == "rekey": + return False + warnings.append(label) + return True + + +def _probe_rerank( + model: Optional[str], + *, + api_key: Optional[str], + base_url: Optional[str], + rerank_provider: Optional[str], +) -> tuple[bool, str]: + """Real capability probe for a rerank endpoint. Dispatches by provider + protocol (vllm / deepinfra / dashscope). Never raises.""" + import httpx + + if not base_url or not model: + return False, "no base_url or model configured" + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + headers["Content-Type"] = "application/json" + + try: + if rerank_provider == "deepinfra": + url = f"{base_url.rstrip('/')}/{model}" + body: dict = {"queries": ["ping"], "documents": ["pong"]} + elif rerank_provider == "dashscope": + url = f"{base_url.rstrip('/')}/api/v1/services/rerank/text-rerank/text-rerank" + body = { + "model": model, + "input": {"query": "ping", "documents": ["pong"]}, + "parameters": {"return_documents": False, "top_n": 1}, + } + else: # vllm / OpenAI-compat + url = f"{base_url.rstrip('/')}/rerank" + body = {"model": model, "query": "ping", "documents": ["pong"]} + + with httpx.Client(timeout=15) as client: + resp = client.post(url, json=body, headers=headers) + if resp.status_code != 200: + return False, f"HTTP {resp.status_code}: {resp.text[:200]}" + data = resp.json() + except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc: + return False, f"probe failed: {exc}" + + if rerank_provider == "deepinfra": + scores = data.get("scores") + if isinstance(scores, list) and scores: + return True, "ok" + return False, "endpoint returned no scores" + if rerank_provider == "dashscope": + output = data.get("output") + results = output.get("results") if isinstance(output, dict) else None + if isinstance(results, list) and results: + return True, "ok" + return False, "endpoint returned no results" + # vllm + results = data.get("results") + if isinstance(results, list) and results: + return True, "ok" + return False, "endpoint returned no results" + + +_REQUIRED_EMBEDDING_DIM = 1024 + + +def _probe_embedding_dim(url: str, headers: dict, model: str) -> int | str: + """Try embedding with ``dimensions=1024``; fall back to native dim. + + Returns the effective dimension (int) on success, or an error + description (str) on failure. + """ + import httpx + + def _try_embed(client: httpx.Client, body: dict) -> int | str: + try: + resp = client.post(url, json=body, headers=headers) + if resp.status_code != 200: + return f"HTTP {resp.status_code}" + items = resp.json().get("data", []) + if not items: + return "empty response" + first = items[0] + if not isinstance(first, dict): + return "unexpected response format" + return len(first.get("embedding", [])) + except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc: + return str(exc) + + with httpx.Client(timeout=15) as client: + result = _try_embed( + client, {"model": model, "input": ["dimension check"], "dimensions": _REQUIRED_EMBEDDING_DIM} + ) + if result == _REQUIRED_EMBEDDING_DIM: + return result + return _try_embed(client, {"model": model, "input": ["dimension check"]}) + + +def _verify_embedding_dim( + *, + model: Optional[str], + api_key: Optional[str], + base_url: Optional[str], + non_interactive: bool, +) -> bool: + """Send a test embedding request and verify the vector dimension is 1024. + + Returns True to proceed, False to re-prompt. + """ + if not base_url or not model: + return True + + url = base_url.rstrip("/") + "/embeddings" + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + + while True: + oc.console.print( + oc._t( + " [dim]⏳ Checking embedding dimension…[/dim]", + " [dim]⏳ 正在检测 embedding 维度…[/dim]", + ) + ) + result = _probe_embedding_dim(url, headers, model) + + if result == _REQUIRED_EMBEDDING_DIM: + oc.console.print( + oc._t( + f" [green]✓ Supports {result}-dim.[/green]", + f" [green]✓ 支持 {result} 维。[/green]", + ) + ) + return True + + if isinstance(result, int) and result < _REQUIRED_EMBEDDING_DIM: + oc.console.print( + oc._t( + f" [red]✗ Dimension too small: model outputs {result}-dim, " + f"EverOS requires >= {_REQUIRED_EMBEDDING_DIM}. Please pick another model.[/red]", + f" [red]✗ 维度不足：模型输出 {result} 维，" + f"EverOS 要求 >= {_REQUIRED_EMBEDDING_DIM} 维，请重新选择。[/red]", + ) + ) + return False + + if isinstance(result, int) and result > _REQUIRED_EMBEDDING_DIM: + oc.console.print( + oc._t( + f" [red]✗ Model outputs {result}-dim and does not support the " + f"dimensions parameter to truncate to {_REQUIRED_EMBEDDING_DIM}. " + "Please pick another model.[/red]", + f" [red]✗ 模型输出 {result} 维，且不支持 dimensions 参数" + f"截断到 {_REQUIRED_EMBEDDING_DIM} 维，请重新选择。[/red]", + ) + ) + return False + + oc.console.print( + oc._t( + f" [yellow]✗ Couldn't verify dimension: {result}[/yellow]", + f" [yellow]✗ 无法验证维度：{result}[/yellow]", + ) + ) + if non_interactive: + return False + choice = oc._failure_choice( + [ + (oc._t("Retry", "重试"), "retry"), + (oc._t("Re-enter", "重新选择"), "rekey"), + ], + non_interactive=False, + ) + if choice == "rekey": + return False + + +# Curated OpenAI-compatible endpoints for EverOS memory models. Picking one +# pre-fills its base_url (mirrors the main provider step); everything else is +# reachable via "reuse an existing endpoint" or "custom" (type a base_url). +# These are the providers' documented OpenAI-compatible /v1 endpoints. +_EVEROS_PROVIDERS: list[dict[str, Any]] = [ + { + "name": "openai", + "label": "OpenAI", + "label_zh": "OpenAI", + "base_url": "https://api.openai.com/v1", + "supports": {"llm", "embedding", "multimodal"}, + }, + { + "name": "openrouter", + "label": "OpenRouter", + "label_zh": "OpenRouter", + "base_url": "https://openrouter.ai/api/v1", + "supports": {"llm", "embedding", "rerank", "multimodal"}, + "rerank_provider": "vllm", + }, + { + "name": "deepseek", + "label": "DeepSeek", + "label_zh": "DeepSeek", + "base_url": "https://api.deepseek.com/v1", + "supports": {"llm"}, + }, + { + "name": "deepinfra", + "label": "DeepInfra", + "label_zh": "DeepInfra", + "base_url": "https://api.deepinfra.com/v1/openai", + "supports": {"llm", "embedding", "rerank"}, + "rerank_provider": "deepinfra", + "rerank_base_url": "https://api.deepinfra.com/v1/inference", + }, + { + "name": "siliconflow", + "label": "SiliconFlow", + "label_zh": "硅基流动 SiliconFlow", + "base_url": "https://api.siliconflow.cn/v1", + "supports": {"llm", "embedding", "rerank"}, + "rerank_provider": "vllm", + }, + { + "name": "dashscope", + "label": "DashScope (Alibaba)", + "label_zh": "阿里百炼 DashScope", + "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", + "supports": {"llm", "embedding", "rerank"}, + "rerank_provider": "dashscope", + "rerank_base_url": "https://dashscope.aliyuncs.com", + }, +] + + +def _match_provider_by_url(base_url: Optional[str]) -> Optional[str]: + """Reverse-lookup a curated provider name from its base_url.""" + if not base_url: + return None + normalized = base_url.rstrip("/") + for prov in _EVEROS_PROVIDERS: + if prov["base_url"].rstrip("/") == normalized: + return prov["name"] + return None + + +# Per-role config: menu/verify label, model-id example, whether optional, and +# whether to run a connectivity probe after configuring (rerank/multimodal use +# non-chat endpoints whose /models probe isn't a reliable health check). +_EVEROS_ROLES: dict[str, dict[str, Any]] = { + "llm": { + "label": ("Memory LLM", "记忆 LLM"), + "example": "gpt-4.1-mini", + "optional": False, + "verify": True, + "purpose": ( + "Reads each conversation to judge what matters and extract the key points.", + "从对话中判断信息边界、抽取要点。", + ), + # Worded as a floor rather than a default: the field is pre-filled with + # the user's own main model, because a recommended id is only reachable + # if their key carries it. This tells them how to judge their own. + "recommendation": ( + "Capability floor: [bold]gpt-4.1-mini[/bold] -- weaker models degrade extraction", + "能力下限参考 [bold]gpt-4.1-mini[/bold]：低于这个水平会明显影响提取质量", + ), + "continue_hint": ("memory extraction may fail", "记忆抽取可能失败"), + }, + "embedding": { + "label": ("Memory embedding", "记忆 embedding"), + "example": "Qwen/Qwen3-Embedding-4B", + # Optional in the sense that memory still functions without it: the + # adapter drops to KEYWORD search, which needs no vectors. Strongly + # advised all the same -- lexical recall misses a memory the moment the + # user phrases the question differently. + "optional": True, + "verify": True, + "purpose": ( + "Turns text into vectors so memories are found by meaning, not just keywords.", + "把文字转成向量，让记忆能按「意思」检索，而不只是按关键词。", + ), + "tag": ( + "[accent](optional, strongly advised)[/accent]", + "[accent]（可选，强烈建议配置）[/accent]", + ), + "cost": ( + "Without it: rephrase a question and it may miss a memory you have;\n recall can only match keywords.", + "不配置：换个说法提问就可能找不到已有记忆，记忆召回时只能使用关键词检索。", + ), + "recommendation": ( + "Recommended: [bold]Qwen/Qwen3-Embedding-4B[/bold] -- must be [bold yellow]1024-dim[/bold yellow],\n" + " Chinese + English", + "推荐 [bold]Qwen/Qwen3-Embedding-4B[/bold]，需 [bold yellow]1024 维[/bold yellow]且支持中英文的模型", + ), + "continue_hint": ("semantic recall will be unavailable", "语义召回将不可用"), + "skip_note": ( + " [yellow]! Skipped: recall will match keywords, not meaning.[/yellow]\n" + " [dim]Phrase a question differently and it may miss a memory you have.\n" + " Configure it later, then run `everos cascade backfill`.[/dim]", + " [yellow]⚠ 已跳过：召回将按关键词匹配，而非按语义。[/yellow]\n" + " [dim]换一种说法提问，就可能找不到已有的记忆。\n" + " 日后配好后运行 everos cascade backfill 可为已存记忆补上向量。[/dim]", + ), + }, + "rerank": { + "label": ("Memory rerank", "记忆 rerank"), + "example": "Qwen/Qwen3-Reranker-4B", + "optional": True, + "verify": True, + "purpose": ( + "Re-ranks what semantic search found so the best match comes first, at a small\n latency cost.", + "在语义召回一批候选后再精排一遍，让最相关的排在最前，会略增延迟。", + ), + "tag": ( + "[accent](optional, advised)[/accent]", + "[accent]（可选，建议配置）[/accent]", + ), + "recommendation": ( + "Recommended: [bold]Qwen/Qwen3-Reranker-4B[/bold]", + "推荐 [bold]Qwen/Qwen3-Reranker-4B[/bold]", + ), + "continue_hint": ("rerank quality may degrade", "rerank 精度可能下降"), + "skip_note": ( + " [dim]Skipped rerank; memory retrieval still works.[/dim]", + " [dim]已跳过 rerank，记忆检索仍可用。[/dim]", + ), + }, + "multimodal": { + "label": ("Memory multimodal", "记忆多模态"), + "example": "google/gemini-3-flash-preview", + "optional": True, + "verify": True, + "purpose": ( + "Lets Raven understand and recall images / PDFs / audio as memory.", + "让 Raven 把图片 / PDF / 音频也作为记忆来理解和检索。", + ), + "cost": ( + "Without it: those files stay out of memory. Having such files is not the same\n" + " as needing them remembered -- configure it when you do.", + "不配置：这类文件不进入记忆；有这类文件并不等于需要，确有此需求时再配即可。", + ), + "recommendation": ( + "Recommended: [bold]google/gemini-3-flash-preview[/bold]", + "推荐 [bold]google/gemini-3-flash-preview[/bold]", + ), + "skip_note": ( + " [dim]Skipped; nothing else is affected -- configure it if you come to need\n multimodal memory.[/dim]", + " [dim]已跳过；其余功能不受影响，日后确有把多模态内容纳入记忆的需求时再配即可。[/dim]", + ), + }, +} + + +_EMBEDDING_MODEL_PATTERNS = ("embed", "bge", "e5-", "gte-") +_MULTIMODAL_MODEL_PATTERNS = ("vision", "4o", "gemini", "pixtral", "qwen-vl", "qwen2-vl", "qwen2.5-vl") + + +def _fetch_everos_models( + base_url: Optional[str], + api_key: Optional[str], + *, + section: str = "llm", + provider_name: Optional[str] = None, +) -> Optional[list[str]]: + """Fetch available model ids from a provider endpoint. Never raises. + + For ``section="embedding"``, delegates to per-provider logic because + each provider exposes embedding models differently. + """ + if not base_url: + return None + if section == "embedding": + return _fetch_embedding_models(base_url, api_key, provider_name) + if section == "rerank": + return _fetch_rerank_models(base_url, api_key, provider_name) + if section == "multimodal": + return _fetch_multimodal_models(base_url, api_key, provider_name) + return _fetch_openai_models(base_url, api_key) + + +def _fetch_openai_models( + base_url: str, + api_key: Optional[str], + *, + params: Optional[dict[str, str]] = None, +) -> Optional[list[str]]: + """``GET {base_url}/models`` with OpenAI-style response parsing.""" + import httpx + + url = base_url.rstrip("/") + "/models" + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + try: + with httpx.Client(timeout=10) as client: + resp = client.get(url, headers=headers, params=params) + if resp.status_code != 200: + return None + data = resp.json() + except (httpx.HTTPError, httpx.InvalidURL, ValueError): + return None + items = data.get("data") if isinstance(data, dict) else None + if not isinstance(items, list): + return None + ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")] + return sorted(ids) or None + + +def _fetch_deepinfra_models( + api_key: Optional[str], + reported_type: str, + *, + name_contains: Optional[str] = None, +) -> Optional[list[str]]: + """Fetch DeepInfra models filtered by ``reported_type`` and optional name substring.""" + import httpx + + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + try: + with httpx.Client(timeout=10) as client: + resp = client.get("https://api.deepinfra.com/models/list", headers=headers) + if resp.status_code != 200: + return None + data = resp.json() + except (httpx.HTTPError, ValueError): + return None + items = data if isinstance(data, list) else [] + ids = [ + m.get("model_name") + for m in items + if isinstance(m, dict) + and m.get("reported_type") == reported_type + and m.get("model_name") + and (name_contains is None or name_contains in m.get("model_name", "")) + ] + return sorted(ids) or None + + +def _fetch_embedding_models( + base_url: str, + api_key: Optional[str], + provider_name: Optional[str], +) -> Optional[list[str]]: + """Provider-specific embedding model listing.""" + if provider_name == "openrouter": + return _fetch_openai_models(base_url.rstrip("/") + "/embeddings", api_key) + + if provider_name == "siliconflow": + return _fetch_openai_models(base_url, api_key, params={"type": "text", "sub_type": "embedding"}) + + if provider_name == "deepinfra": + return _fetch_deepinfra_models(api_key, "embeddings") + + # OpenAI, DashScope, custom — GET /models + name-based filter. + ids = _fetch_openai_models(base_url, api_key) + if ids is None: + return None + filtered = [i for i in ids if any(p in i.lower() for p in _EMBEDDING_MODEL_PATTERNS)] + return filtered or None + + +def _fetch_rerank_models( + base_url: str, + api_key: Optional[str], + provider_name: Optional[str], +) -> Optional[list[str]]: + """Provider-specific rerank model listing.""" + if provider_name == "deepinfra": + # The deepinfra provider hardcodes a Qwen3-Reranker chat template, + # so only Qwen3-Reranker models are compatible. + return _fetch_deepinfra_models(api_key, "reranker", name_contains="Qwen3-Reranker") + + if provider_name == "siliconflow": + return _fetch_openai_models(base_url, api_key, params={"sub_type": "reranker"}) + + if provider_name == "dashscope": + return ["gte-rerank-v2"] + + if provider_name == "openrouter": + return _fetch_openai_models(base_url, api_key, params={"output_modalities": "rerank"}) + + # vllm / custom — no standard rerank listing. + return None + + +def _fetch_multimodal_models( + base_url: str, + api_key: Optional[str], + provider_name: Optional[str], +) -> Optional[list[str]]: + """Provider-specific multimodal (vision) model listing.""" + if provider_name == "openrouter": + return _fetch_openai_models(base_url, api_key, params={"input_modalities": "image"}) + + # OpenAI, custom — GET /models + name-based filter. + ids = _fetch_openai_models(base_url, api_key) + if ids is None: + return None + filtered = [i for i in ids if any(p in i.lower() for p in _MULTIMODAL_MODEL_PATTERNS)] + return filtered or None + + +def _match_everos_default(example: str, models: list[str]) -> str: + """Find the best match for ``example`` in the fetched model list. + + The example (e.g. ``gpt-4.1-mini``) is a bare model name, while + ``models`` may carry provider prefixes (``openai/gpt-4.1-mini``). + Returns the first model whose id ends with ``/example`` or equals + ``example`` exactly; falls back to the bare example string so the + autocomplete input is pre-filled even if no exact match exists. + """ + lower = example.lower() + suffix = f"/{lower}" + for mid in models: + if mid.lower() == lower or mid.lower().endswith(suffix): + return mid + return example + + +def _preferred_memory_model(section: str, main_model: Optional[str], chosen_provider: Optional[str]) -> Optional[str]: + """The main chat model, when it is a sensible pre-fill for this role. + + Only the llm role -- an embedding / rerank / multimodal endpoint does not + serve a chat model. Only when the picked provider is the main model's own: no + other provider carries that id, and pre-filling one it cannot serve turns + Enter into a verification failure. A custom endpoint has no resolved provider + and is left alone for the same reason. + """ + if section != "llm" or not main_model or chosen_provider is None: + return None + if chosen_provider != _resolve_model_provider(main_model): + return None + return _resolve_reuse_llm_creds(main_model).get("model") + + +def _everos_pick_model( + *, + base_url: Optional[str], + api_key: Optional[str], + example: str, + allow_back: bool, + section: str = "llm", + provider_name: Optional[str] = None, + recommendation: Optional[tuple[str, str]] = None, + preferred: Optional[str] = None, +) -> Any: + """Pick a model id for an EverOS endpoint: fetch ``/models`` for a + fuzzy-searchable list, else fall back to free text. Empty submit = back. + + ``preferred`` pre-fills a model the user is already known to have access to + -- their main chat model. It wins over ``example`` because a recommended + model is only a recommendation if the user's key can reach it, and many keys + cannot; ``example`` then reads as the capability floor rather than the + default (see ``recommendation``). + """ + questionary = oc._require_questionary() + from raven.cli._styles import RAVEN_STYLE + + oc.console.print(oc._t(" [dim]⏳ Loading models…[/dim]", " [dim]⏳ 正在拉取模型列表…[/dim]")) + models = _fetch_everos_models(base_url, api_key, section=section, provider_name=provider_name) + if preferred: + oc.console.print( + oc._t( + f" [dim]Pre-filled with your main model [bold]{preferred}[/bold] -- press Enter to accept.[/dim]", + f" [dim]已填入你的主模型 [bold]{preferred}[/bold]，直接回车即可。[/dim]", + ) + ) + if recommendation: + oc.console.print(f" [dim]{oc._t(*recommendation)}[/dim]") + if models: + default_model = preferred or _match_everos_default(example, models) + question = questionary.autocomplete( + oc._t( + f"Model ({len(models)} available — type to filter):", + f"模型(共 {len(models)} 个 — 输入可筛选):", + ), + choices=models, + default=default_model, + ignore_case=True, + match_middle=True, + placeholder=oc._back_placeholder(allow_back), + style=RAVEN_STYLE, + qmark=oc._QMARK, + ) + # Trigger the completion popup immediately so the user sees + # all available models without typing first. + app = question.application + + def _show_completions() -> None: + buf = app.current_buffer + buf.start_completion() + + app.pre_run_callables.append(_show_completions) + chosen = question.ask() + else: + oc.console.print( + oc._t( + " [dim]Couldn't list models from this endpoint — type the id manually.[/dim]", + " [dim]该端点拉不到模型列表 — 请手动输入模型 id。[/dim]", + ) + ) + chosen = questionary.text( + oc._t(f"Model id (e.g. {example}):", f"模型 id（如 {example}）："), + default=preferred or "", + placeholder=oc._back_placeholder(allow_back), + style=RAVEN_STYLE, + qmark=oc._QMARK, + ).ask() + if chosen is None: + raise typer.Exit(1) + chosen = chosen.strip() + if allow_back and chosen == "": + return oc._BACK + if not chosen: + raise typer.Exit(1) + return chosen + + +def _everos_pick_creds_and_model( + *, + section: str, + example: str, + main_model: Optional[str], + non_interactive: bool, + recommendation: Optional[tuple[str, str]] = None, +) -> Any: + """Mirror the main provider step for one EverOS model: pick a source + (curated provider / custom) → API key → model. Returns a dict with + ``model`` / ``api_key`` / ``base_url`` (plus ``provider`` for rerank), or + ``oc._BACK`` when the user backs out of the source picker. Empty submit on any + field rewinds one step.""" + questionary = oc._require_questionary() + from raven.cli._styles import RAVEN_STYLE + + llm_section = _everos_section("llm") + + # For the LLM role, default to the main chat model's provider. + # For other roles (embedding/rerank/multimodal), default to whichever + # provider the LLM step just configured — the user likely has the + # same API key and only needs to pick a different model. + if section == "llm": + default_provider = _resolve_model_provider(main_model or "") + reuse_source = "main" + else: + default_provider = _match_provider_by_url(llm_section.get("base_url")) + reuse_source = "llm" + + while True: # source picker — a field-level back rewinds here + choices: list[Any] = [] + default_choice = None + for prov in _EVEROS_PROVIDERS: + if section not in prov.get("supports", set()): + continue + is_default = default_provider is not None and prov["name"] == default_provider + if is_default: + if reuse_source == "main": + label = oc._t( + f"{prov['label']} (main model provider, reuse Key)", + f"{prov['label_zh']}（主模型服务商，复用 Key）", + ) + else: + label = oc._t( + f"{prov['label']} (memory LLM provider, reuse Key)", + f"{prov['label_zh']}（记忆 LLM 服务商，复用 Key）", + ) + else: + label = oc._t(prov["label"], prov["label_zh"]) + choice = questionary.Choice(label, value=("provider", prov)) + choices.append(choice) + if is_default: + default_choice = choice.value + choices.append( + questionary.Choice( + oc._t("Other (custom OpenAI-compatible endpoint)", "其他(自定义 OpenAI 兼容端点)"), + value=("custom",), + ) + ) + choices.append(questionary.Separator()) + choices.append(questionary.Choice(oc._t("Back", "返回"), value=oc._BACK)) + + src = questionary.select( + oc._t("Pick a provider (or reuse / custom):", "选择服务商(或复用 / 自定义):"), + choices=choices, + default=default_choice, + style=RAVEN_STYLE, + qmark=oc._QMARK, + ).ask() + if src is None: + raise typer.Exit(1) + if src is oc._BACK: + return oc._BACK + kind = src[0] + + # Resolve (api_key, base_url) from the chosen source. + chosen_provider: Optional[str] = None + if kind == "provider": + chosen_provider = src[1]["name"] + base_url = src[1]["base_url"] + prefilled_key: Optional[str] = None + if default_provider == src[1]["name"]: + if reuse_source == "main": + prefilled_key = _resolve_reuse_llm_creds(main_model or "").get("api_key") + else: + prefilled_key = llm_section.get("api_key") + if prefilled_key: + if reuse_source == "main": + oc.console.print( + oc._t( + " [dim]API key reused from main chat model.[/dim]", + " [dim]已复用主对话模型的 API Key。[/dim]", + ) + ) + else: + oc.console.print( + oc._t( + " [dim]API key reused from memory LLM.[/dim]", + " [dim]已复用记忆 LLM 的 API Key。[/dim]", + ) + ) + api_key = prefilled_key + else: + api_key = oc._prompt_api_key(src[1]["name"], allow_back=True) + if api_key is oc._BACK: + continue + else: # custom + base_url = _prompt_text(oc._t("Base URL (must include /v1):", "Base URL(需包含 /v1):"), allow_back=True) + if base_url is oc._BACK: + continue + api_key = _prompt_text(oc._t("API key (hidden):", "API Key(隐藏输入):"), secret=True, allow_back=True) + if api_key is oc._BACK: + continue + + # Guard against a source that resolved to an empty key / endpoint — + # set_everos_section drops None values, which would otherwise persist a + # section with a model but no usable endpoint. + if not (api_key and base_url): + oc.console.print( + oc._t( + " [yellow]✗ Missing API key or Base URL for this source — pick another.[/yellow]", + " [yellow]✗ 该来源缺少 API Key 或 Base URL — 请换一个。[/yellow]", + ) + ) + continue + + # rerank: resolve service type + override base_url when needed. + rerank_provider: Optional[str] = None + if section == "rerank": + chosen_prov_dict = src[1] if kind == "provider" else None + if chosen_prov_dict and chosen_prov_dict.get("rerank_provider"): + rerank_provider = chosen_prov_dict["rerank_provider"] + if chosen_prov_dict.get("rerank_base_url"): + base_url = chosen_prov_dict["rerank_base_url"] + else: + rerank_provider = questionary.select( + oc._t("Rerank service type:", "rerank 服务类型:"), + choices=[ + questionary.Choice("deepinfra", value="deepinfra"), + questionary.Choice("vllm", value="vllm"), + questionary.Choice("dashscope", value="dashscope"), + questionary.Choice(oc._t("Back", "返回"), value=oc._BACK), + ], + style=RAVEN_STYLE, + qmark=oc._QMARK, + ).ask() + if rerank_provider is None: + raise typer.Exit(1) + if rerank_provider is oc._BACK: + continue + + model = _everos_pick_model( + base_url=base_url, + api_key=api_key, + example=example, + allow_back=True, + section=section, + provider_name=chosen_provider, + recommendation=recommendation, + preferred=_preferred_memory_model(section, main_model, chosen_provider), + ) + if model is oc._BACK: + continue + + result: dict[str, Any] = {"model": model, "api_key": api_key, "base_url": base_url} + if rerank_provider: + result["provider"] = rerank_provider + return result + + +def _config_everos_role( + *, section: str, main_model: Optional[str], non_interactive: bool, warnings: list[str], skip_test: bool = False +) -> Any: + """Configure one EverOS memory role (llm / embedding / rerank / multimodal) + with the unified provider→key→model flow, reuse shortcuts, and a back loop. + + Returns ``None`` normally; returns ``oc._ABORT_EVEROS`` when the user gives up a + required role (the caller then disables EverOS, leaving no long-term memory).""" + questionary = oc._require_questionary() + from raven.cli._styles import RAVEN_STYLE + from raven.config.update_everos import clear_everos_section, set_everos_section + + role = _EVEROS_ROLES[section] + label_en, label_zh = role["label"] + purpose_en, purpose_zh = role["purpose"] + optional = role["optional"] + verify_label = oc._t(label_en, label_zh) + + # Tell the user what this model is for, and what skipping it costs, before + # asking them to configure it. Header sits on the 2-space info column (bold + # accent); purpose and cost nest under it, matching the layout used + # everywhere else. + # + # The cost line is dim rather than a warning colour on purpose: this is + # pre-decision information, and colouring it would cry wolf before the user + # has chosen anything. The warning comes after, from ``skip_note``. + # + # Roles that want to be configured say so in their own ``tag`` -- calling all + # three merely "optional" flattens the difference between losing semantic + # recall entirely and losing a little ranking accuracy. + tag_markup = oc._t(*role["tag"]) if role.get("tag") else oc._t("[dim](optional)[/dim]", "[dim]（可选）[/dim]") + lines = [f" [bold][accent]{oc._t(label_en, label_zh)}[/accent][/bold]" + (f" {tag_markup}" if optional else "")] + lines.append(f" [dim]{oc._t(purpose_en, purpose_zh)}[/dim]") + if role.get("cost"): + lines.append(f" [dim]{oc._t(*role['cost'])}[/dim]") + oc.console.print() + # highlight=False so Rich's default highlighter doesn't tint the dim prose + # (parens/numbers/words) and make an informational hint read like an error. + oc.console.print("\n".join(lines), highlight=False) + + while True: # role-menu loop — a back-out of the source picker returns here + current = _everos_section(section).get("model") if _everos_role_configured(section) else None + if current: + choices = [ + questionary.Choice(oc._t(f"Keep current: {current}", f"沿用当前:{current}"), value="keep"), + questionary.Choice(oc._t("Reconfigure", "重新配置"), value="redo"), + ] + if optional: + choices.append(questionary.Choice(oc._t("Skip", "跳过"), value="off")) + action = questionary.select( + oc._t("Already configured — what now?", "已配置,怎么处理?"), + choices=choices, + style=RAVEN_STYLE, + qmark=oc._QMARK, + ).ask() + if action is None: + raise typer.Exit(1) + if action == "keep": + return + if action == "off": + clear_everos_section(section) + oc.console.print(oc._t(f" [dim]{label_en} skipped.[/dim]", f" [dim]已跳过 {label_zh}。[/dim]")) + return + elif optional: + action = questionary.select( + oc._t("Configure it?", "要配置吗?"), + choices=[ + questionary.Choice(oc._t("Configure", "配置"), value="redo"), + questionary.Choice(oc._t("Skip", "跳过"), value="skip"), + ], + style=RAVEN_STYLE, + qmark=oc._QMARK, + ).ask() + if action is None: + raise typer.Exit(1) + if action == "skip": + # Printed verbatim rather than wrapped in [dim]: skipping rerank + # costs ordering, skipping embedding costs semantic recall + # entirely, and one of those deserves to be seen. + note_en, note_zh = role.get( + "skip_note", (f" [dim]Skipped {label_en}.[/dim]", f" [dim]已跳过 {label_zh}。[/dim]") + ) + oc.console.print(oc._t(note_en, note_zh), highlight=False) + return + # A required role with nothing configured falls straight into the picker. + + result = _everos_pick_creds_and_model( + section=section, + example=role["example"], + main_model=main_model, + non_interactive=non_interactive, + recommendation=role.get("recommendation"), + ) + if result is oc._BACK: + if optional or _everos_role_configured(section): + # Optional roles offer Skip; a required role already configured + # falls back to its keep/reconfigure menu. Either way, re-show + # the role menu rather than forcing the give-up exit. + continue + # A required role with nothing configured has no Skip, so backing out + # of the picker would loop forever. Offer a bounded exit -- keep + # trying, or leave without long-term memory. Stated in full and in + # colour: this is the only place the wizard can lose memory + # altogether, and "no cross-session memory" is a consequence a user + # should not discover weeks later by noticing the agent forgets + # everything. + oc.console.print() + oc.console.print( + oc._t( + f" [yellow]⚠ {label_en} is required for long-term memory.[/yellow]\n" + " [dim]Without it Raven has no memory across sessions: every conversation starts\n" + " from nothing, with no recollection of your preferences or of what was done before.[/dim]", + f" [yellow]⚠ {label_zh} 是长期记忆的必需项。[/yellow]\n" + " [dim]放弃后 Raven 没有任何跨会话记忆：每次对话都从零开始，不记得你的偏好，\n" + " 也不记得之前做过什么。[/dim]", + ), + highlight=False, + ) + action = questionary.select( + oc._t("What would you like to do?", "想做什么？"), + choices=[ + questionary.Choice(oc._t("Pick a provider / model", "选择服务商 / 模型"), value="retry"), + questionary.Choice( + oc._t("Give up (no long-term memory)", "放弃（不启用长期记忆）"), + value="abort", + ), + ], + style=RAVEN_STYLE, + qmark=oc._QMARK, + ).ask() + if action is None: + raise typer.Exit(1) + if action == "retry": + continue + return oc._ABORT_EVEROS + + if role["verify"] and skip_test: + oc.console.print( + oc._t( + f" [dim]Skipping the {verify_label} test call (--skip-test).[/dim]", + f" [dim]已跳过 {verify_label} 的测试调用(--skip-test)。[/dim]", + ) + ) + ok = True + elif section == "llm": + ok = _verify_everos_llm( + verify_label, + model=result["model"], + api_key=result["api_key"], + base_url=result["base_url"], + non_interactive=non_interactive, + warnings=warnings, + continue_hint=role.get("continue_hint"), + ) + elif section == "embedding": + ok = _verify_embedding_dim( + model=result["model"], + api_key=result["api_key"], + base_url=result["base_url"], + non_interactive=non_interactive, + ) + elif section == "rerank": + ok = _verify_rerank( + verify_label, + model=result["model"], + api_key=result["api_key"], + base_url=result["base_url"], + rerank_provider=result.get("provider"), + non_interactive=non_interactive, + warnings=warnings, + continue_hint=role.get("continue_hint"), + ) + elif section == "multimodal": + ok = _verify_everos_llm( + verify_label, + model=result["model"], + api_key=result["api_key"], + base_url=result["base_url"], + non_interactive=non_interactive, + warnings=warnings, + continue_hint=role.get("continue_hint"), + ) + else: + ok = True + if not ok: + continue + + set_everos_section(section, result) + oc.console.print( + oc._t( + f" [green]✓ {label_en} configured.[/green]", + f" [green]✓ 已配置 {label_zh}。[/green]", + ) + ) + return + + +def _step4_memory( + *, skip: bool, non_interactive: bool, main_model: Optional[str], warnings: list[str], skip_test: bool = False +) -> object: + """Step 4 -- EverOS long-term memory (model sub-screens). + + The bootstrap seeds ``memory.backend="everos"`` (schema default) and everos + is the only memory backend, so this step does not ask whether to enable it: + it either confirms the seed by configuring the llm role, or resolves it back + to ``None`` on skip / non-interactive / give-up. ``None`` means no long-term + memory at all, not a fallback to something simpler. + + ``_memory_enabled`` gates on the llm role alone, so a fresh modelless seed + reads as "not configured yet" and the keep/reconfigure menu only appears once + that model is actually on disk. embedding and rerank are offered here but + never gate: skipping them costs recall quality, not memory itself. + """ + oc._step_header(4, oc._t("EverOS long-term memory", "EverOS 长期记忆")) + + import sys + + if sys.platform == "win32": + oc.console.print( + oc._t( + " [yellow]⚠ EverOS memory engine does not support native Windows.[/yellow]\n" + " [dim]Run Raven inside WSL for full memory support.[/dim]\n" + " [dim]Skipping memory configuration.[/dim]", + " [yellow]⚠ EverOS 记忆引擎暂不支持 Windows 原生环境。[/yellow]\n" + " [dim]在 WSL 中运行 Raven 可获得完整记忆支持。[/dim]\n" + " [dim]已跳过记忆配置。[/dim]", + ) + ) + _set_memory_backend(None) + return None + + if skip or non_interactive: + # Never configured the required models here → disable backend-driven + # memory so runtime doesn't activate EverOS without an llm/embedding. + # (``_memory_enabled`` already gates on both required models, so an + # already-enabled+configured setup is preserved.) + if not _memory_enabled(): + _set_memory_backend(None) + oc.console.print( + oc._t( + " [dim]Long-term memory stays off.[/dim]", + " [dim]长期记忆保持关闭。[/dim]", + ) + ) + return None + + questionary = oc._require_questionary() + from raven.cli._styles import RAVEN_STYLE + + if _memory_enabled(): + action = questionary.select( + oc._t( + "EverOS long-term memory is already enabled. What would you like to do?", + "EverOS 长期记忆已启用。想做什么?", + ), + choices=[ + questionary.Choice(oc._t("Keep it enabled", "保持启用"), value="keep"), + questionary.Choice(oc._t("Reconfigure", "重新配置"), value="redo"), + ], + style=RAVEN_STYLE, + qmark=oc._QMARK, + ).ask() + if action is None: + raise typer.Exit(1) + if action == "keep": + return None # backend already "everos" + models on disk; leave as-is + else: + # No enable/decline question: everos is the only memory backend, so the + # step goes straight into configuring it. Leaving is still possible -- + # backing out of the required roles reaches the give-up prompt, which + # spells out what is lost. + # Wrapped by hand: rich re-wraps at the terminal width and drops the + # two-space indent on continuation lines, which reads as a stray + # left-flush sentence under an indented block. + oc.console.print( + oc._t( + " [dim]Raven's long-term memory comes from EverOS. What it can do grows with\n" + " what you configure:[/dim]\n" + " [dim] memory LLM only conversations become memories; recall matches keywords[/dim]\n" + " [dim] + memory embedding recall matches meaning, not wording (strongly advised)[/dim]\n" + " [dim] + memory rerank recall ordering gets sharper[/dim]", + " [dim]Raven 拥有 EverOS 提供的强大长期记忆能力，能力随配置递进：[/dim]\n" + " [dim] 仅记忆 LLM 对话会被提炼成记忆存下来，召回按关键词匹配[/dim]\n" + " [dim] + 记忆 embedding 召回按语义匹配，换个问法也能找到（强烈建议配）[/dim]\n" + " [dim] + 记忆 rerank 召回结果排序更准[/dim]", + ), + highlight=False, + ) + + # Ensure the EverOS home directory has its config templates (everos.toml + # + ome.toml) BEFORE writing model sections — set_everos_section merges + # into the template so default sections (memory/sqlite/lancedb/api) are + # preserved. Also creates ome.toml which the runtime requires. + from raven.config.update_everos import configure_everos_env, ensure_everos_home + + configure_everos_env() + ensure_everos_home() + + # Configure required models FIRST, then flip the backend on — so a Ctrl+C + # mid-configuration leaves backend at its prior (disabled) value rather + # than an enabled-but-modelless state. + for _role in ("llm", "embedding", "rerank", "multimodal"): + # Each role prints one leading blank before its own header, so no extra + # separator here — avoids the double blank line between roles. + outcome = _config_everos_role( + section=_role, + main_model=main_model, + non_interactive=non_interactive, + warnings=warnings, + skip_test=skip_test, + ) + if outcome is oc._ABORT_EVEROS: + _set_memory_backend(None) + oc.console.print( + oc._t( + " [yellow]⚠ Gave up long-term memory: Raven will not remember anything " + "between sessions.[/yellow]\n" + " [dim]Run `raven onboard` again whenever you want to configure it.[/dim]", + " [yellow]⚠ 已放弃长期记忆，Raven 不会记住任何跨会话内容。[/yellow]\n" + " [dim]随时可以重新运行 raven onboard 配置。[/dim]", + ) + ) + return None + + # Verify EverOS server is reachable (auto-starts if needed) + import asyncio + + from raven.plugin.memory.everos._server import ensure_everos_server + + oc.console.print() + oc.console.print( + oc._t( + " [dim]Starting EverOS service...[/dim]", + " [dim]正在启动 EverOS 服务...[/dim]", + ) + ) + try: + asyncio.run(ensure_everos_server()) + oc.console.print( + oc._t( + " [green]✓ EverOS service is running.[/green]", + " [green]✓ EverOS 服务已启动。[/green]", + ) + ) + except RuntimeError as exc: + oc.console.print( + oc._t( + f" [red]✗ EverOS service failed to start: {exc}[/red]\n" + " [dim]Check: everos installed? Port 18791 free? " + "See ~/.raven/logs/everos-server.log[/dim]", + f" [red]✗ EverOS 服务启动失败：{exc}[/red]\n" + " [dim]请检查：everos 是否安装？端口 18791 是否被占用？" + "查看 ~/.raven/logs/everos-server.log[/dim]", + ) + ) + retry = questionary.select( + oc._t("What to do?", "怎么办？"), + choices=[ + questionary.Choice(oc._t("Retry", "重试"), value="retry"), + questionary.Choice(oc._t("Skip (memory disabled)", "跳过（记忆禁用）"), value="skip"), + ], + style=RAVEN_STYLE, + qmark=oc._QMARK, + ).ask() + if retry == "retry": + # Recurse once — the loop in _step4_memory handles further retries + try: + asyncio.run(ensure_everos_server()) + oc.console.print( + oc._t( + " [green]✓ EverOS service is running.[/green]", + " [green]✓ EverOS 服务已启动。[/green]", + ) + ) + except RuntimeError: + oc.console.print( + oc._t( + " [red]✗ Still failed. Disabling memory.[/red]", + " [red]✗ 仍然失败。禁用记忆功能。[/red]", + ) + ) + _set_memory_backend(None) + return None + else: + _set_memory_backend(None) + return None + _report_everos_capabilities() + _set_memory_backend("everos") + return None + + +def _report_everos_capabilities() -> None: + """Say what the running server can actually do, not just that it answers. + + ``ensure_everos_server`` proves the process is up and nothing more. Since + everos 1.2.1 a server whose embedding provider failed to build still answers + 200 and degrades to keyword-only search, so stopping at "running" would + print a tick over an install that cannot recall anything. The roles were + each verified against their provider earlier in this step; what is new here + is whether everos itself could build them from what got written to + ``everos.toml``. + + Silent on a server too old to report capabilities -- reading that silence as + "unavailable" would condemn a working install. + """ + from raven.config.raven import load_raven_config + from raven.plugin.memory.everos._health import ( + DEGRADING_SECTIONS, + REQUIRED_SECTIONS, + configured_base_url, + probe_capabilities, + ) + + # The configured address, not the default: probing the wrong port reports on + # a server nobody is using, and reads as "not running". + report = probe_capabilities(configured_base_url(load_raven_config())) + if not report.reports_capabilities: + return + configured = [s for s in (*REQUIRED_SECTIONS, *DEGRADING_SECTIONS) if _everos_role_configured(s)] + broken = [s for s in configured if report.available(s) is False] + if not broken: + names = " and ".join(configured) + oc.console.print( + oc._t( + f" [green]✓ {names} {'is' if len(configured) == 1 else 'are'} available.[/green]", + f" [green]✓ {names} 均可用。[/green]", + ) + ) + return + names = " and ".join(broken) + oc.console.print( + oc._t( + f" [yellow]⚠ {names} is configured but EverOS could not build it.[/yellow]\n" + " [dim]Memory runs degraded until this is fixed.[/dim]\n" + f" [dim]Check: {_everos_server_log_hint()}[/dim]", + f" [yellow]⚠ {names} 已配置，但 EverOS 未能构建成功。[/yellow]\n" + " [dim]在此修复前，记忆能力将处于降级状态。[/dim]\n" + f" [dim]请查看：{_everos_server_log_hint()}[/dim]", + ) + ) + + +def _everos_server_log_hint() -> str: + from raven.plugin.memory.everos._server import server_log_path + + return str(server_log_path()) diff --git a/tests/test_cli_onboard_commands.py b/tests/test_cli_onboard_commands.py index 150dd472..0a51f3e1 100644 --- a/tests/test_cli_onboard_commands.py +++ b/tests/test_cli_onboard_commands.py @@ -24,7 +24,7 @@ import typer from typer.testing import CliRunner -from raven.cli import onboard_commands +from raven.cli import onboard_commands, onboard_everos from raven.cli.commands import app from raven.config.loader import set_config_path @@ -652,7 +652,7 @@ def test_onboard_interactive_uses_stubbed_pickers( # interactive Step 1 path can be asserted without driving every screen. monkeypatch.setattr(onboard_commands, "_step2_sandbox", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step3_channel", lambda **_: None) - monkeypatch.setattr(onboard_commands, "_step4_memory", lambda **_: None) + monkeypatch.setattr(onboard_everos, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) @@ -788,7 +788,7 @@ def _fake_autocomplete(message, choices, default=None, **kwargs): monkeypatch.setattr(questionary, "autocomplete", _fake_autocomplete) monkeypatch.setattr(onboard_commands, "_step2_sandbox", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step3_channel", lambda **_: None) - monkeypatch.setattr(onboard_commands, "_step4_memory", lambda **_: None) + monkeypatch.setattr(onboard_everos, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) @@ -1294,7 +1294,7 @@ def ask(self): return next(answers) monkeypatch.setattr(questionary, "select", lambda *a, **kw: _FQ()) - onboard_commands._step4_memory(skip=False, non_interactive=False, main_model="openai/gpt-4o-mini", warnings=[]) + onboard_everos._step4_memory(skip=False, non_interactive=False, main_model="openai/gpt-4o-mini", warnings=[]) data = json.loads(tmp_env.read_text()) assert data["memory"]["backend"] is None assert not everos_isolated.exists() @@ -1319,7 +1319,7 @@ def ask(self): return next(answers) monkeypatch.setattr(questionary, "select", lambda *a, **kw: _FQ()) - onboard_commands._step4_memory(skip=False, non_interactive=False, main_model="openai/gpt-4o-mini", warnings=[]) + onboard_everos._step4_memory(skip=False, non_interactive=False, main_model="openai/gpt-4o-mini", warnings=[]) out = " ".join(capsys.readouterr().out.split()) assert "no memory across sessions" in out @@ -1354,7 +1354,7 @@ def ask(self): return next(answers) monkeypatch.setattr(questionary, "select", lambda *a, **kw: _FQ()) - onboard_commands._step4_memory(skip=False, non_interactive=False, main_model="openai/gpt-4o-mini", warnings=[]) + onboard_everos._step4_memory(skip=False, non_interactive=False, main_model="openai/gpt-4o-mini", warnings=[]) out = " ".join(capsys.readouterr().out.split()) for needle in needles: @@ -1396,9 +1396,9 @@ def ask(self): monkeypatch.setattr(questionary, "text", lambda *a, **kw: _FQ(next(text_answers))) monkeypatch.setattr(questionary, "password", lambda *a, **kw: _FQ(next(password_answers))) # No network: model list can't be fetched → free-text entry; probe succeeds. - monkeypatch.setattr(onboard_commands, "_fetch_everos_models", lambda *a, **kw: None) - monkeypatch.setattr(onboard_commands, "_probe_everos_chat", lambda *a, **kw: (True, "ok")) - monkeypatch.setattr(onboard_commands, "_verify_embedding_dim", lambda **kw: True) + monkeypatch.setattr(onboard_everos, "_fetch_everos_models", lambda *a, **kw: None) + monkeypatch.setattr(onboard_everos, "_probe_everos_chat", lambda *a, **kw: (True, "ok")) + monkeypatch.setattr(onboard_everos, "_verify_embedding_dim", lambda **kw: True) import raven.plugin.memory.everos._server as everos_server @@ -1407,7 +1407,7 @@ async def _fake_ensure_everos_server(*a: object, **kw: object) -> None: monkeypatch.setattr(everos_server, "ensure_everos_server", _fake_ensure_everos_server) - onboard_commands._step4_memory( + onboard_everos._step4_memory( skip=False, non_interactive=False, main_model="openrouter/anthropic/claude-sonnet-4-5", @@ -1455,9 +1455,9 @@ def ask(self): monkeypatch.setattr(questionary, "select", lambda *a, **kw: _FQ(next(select_answers))) monkeypatch.setattr(questionary, "text", lambda *a, **kw: _FQ(next(text_answers))) monkeypatch.setattr(questionary, "password", lambda *a, **kw: _FQ(next(password_answers))) - monkeypatch.setattr(onboard_commands, "_fetch_everos_models", lambda *a, **kw: None) - monkeypatch.setattr(onboard_commands, "_probe_everos_chat", lambda *a, **kw: (True, "ok")) - monkeypatch.setattr(onboard_commands, "_verify_embedding_dim", lambda **kw: True) + monkeypatch.setattr(onboard_everos, "_fetch_everos_models", lambda *a, **kw: None) + monkeypatch.setattr(onboard_everos, "_probe_everos_chat", lambda *a, **kw: (True, "ok")) + monkeypatch.setattr(onboard_everos, "_verify_embedding_dim", lambda **kw: True) import raven.plugin.memory.everos._server as everos_server @@ -1467,9 +1467,9 @@ async def _fake_ensure_everos_server(*a: object, **kw: object) -> None: monkeypatch.setattr(everos_server, "ensure_everos_server", _fake_ensure_everos_server) reported: list[int] = [] - monkeypatch.setattr(onboard_commands, "_report_everos_capabilities", lambda: reported.append(1)) + monkeypatch.setattr(onboard_everos, "_report_everos_capabilities", lambda: reported.append(1)) - onboard_commands._step4_memory( + onboard_everos._step4_memory( skip=False, non_interactive=False, main_model="openrouter/anthropic/claude-sonnet-4-5", @@ -1507,10 +1507,10 @@ def ask(self): select_answers = iter([("provider", openai_prov)]) monkeypatch.setattr(questionary, "select", lambda *a, **kw: _FQ(next(select_answers))) monkeypatch.setattr(questionary, "autocomplete", lambda *a, **kw: _FQ("gpt-4.1-mini")) - monkeypatch.setattr(onboard_commands, "_probe_everos_chat", lambda *a, **kw: (True, "ok")) - monkeypatch.setattr(onboard_commands, "_fetch_everos_models", lambda *a, **kw: ["gpt-4.1-mini"]) + monkeypatch.setattr(onboard_everos, "_probe_everos_chat", lambda *a, **kw: (True, "ok")) + monkeypatch.setattr(onboard_everos, "_fetch_everos_models", lambda *a, **kw: ["gpt-4.1-mini"]) - onboard_commands._config_everos_role( + onboard_everos._config_everos_role( section="llm", main_model="openai/gpt-4o-mini", non_interactive=False, warnings=[] ) with everos_isolated.open("rb") as f: @@ -1546,15 +1546,15 @@ def __init__(self, a): def ask(self): return self._a - deepinfra_prov = next(p for p in onboard_commands._EVEROS_PROVIDERS if p["name"] == "deepinfra") + deepinfra_prov = next(p for p in onboard_everos._EVEROS_PROVIDERS if p["name"] == "deepinfra") # No service-type select needed — curated provider auto-resolves it. select_answers = iter(["redo", ("provider", deepinfra_prov)]) monkeypatch.setattr(questionary, "select", lambda *a, **kw: _FQ(next(select_answers))) monkeypatch.setattr(questionary, "text", lambda *a, **kw: _FQ("rerank-model")) - monkeypatch.setattr(onboard_commands, "_fetch_everos_models", lambda *a, **kw: None) - monkeypatch.setattr(onboard_commands, "_probe_rerank", lambda *a, **kw: (True, "ok")) + monkeypatch.setattr(onboard_everos, "_fetch_everos_models", lambda *a, **kw: None) + monkeypatch.setattr(onboard_everos, "_probe_rerank", lambda *a, **kw: (True, "ok")) - onboard_commands._config_everos_role( + onboard_everos._config_everos_role( section="rerank", main_model="openrouter/anthropic/claude-sonnet-4-5", non_interactive=False, @@ -1572,11 +1572,11 @@ def test_memory_seeded_role_is_not_configured(tmp_env: Path, everos_isolated: Pa """A seeded model with an empty api_key does not count as configured.""" from raven.config.update_everos import set_everos_section - assert onboard_commands._everos_role_configured("llm") is False + assert onboard_everos._everos_role_configured("llm") is False set_everos_section("llm", {"model": "openai/gpt-4.1-mini", "api_key": ""}) - assert onboard_commands._everos_role_configured("llm") is False + assert onboard_everos._everos_role_configured("llm") is False set_everos_section("llm", {"api_key": "sk-real"}) - assert onboard_commands._everos_role_configured("llm") is True + assert onboard_everos._everos_role_configured("llm") is True def test_memory_required_role_back_reaches_give_up_menu( @@ -1608,14 +1608,14 @@ def ask(self) -> object: monkeypatch.setattr(questionary, "select", _FQ) - out = onboard_commands._config_everos_role(section="llm", main_model=None, non_interactive=False, warnings=[]) + out = onboard_everos._config_everos_role(section="llm", main_model=None, non_interactive=False, warnings=[]) assert out is onboard_commands._ABORT_EVEROS assert asked == ["picker", "give-up"] def test_model_openai_compatible_heuristic(tmp_env: Path) -> None: """Compat heuristic gates whether the memory LLM can reuse the main model.""" - f = onboard_commands._model_is_openai_compatible + f = onboard_everos._model_is_openai_compatible assert f("openai/gpt-4o-mini") assert f("openrouter/anthropic/claude-sonnet-4-5") assert f("deepseek/deepseek-chat") @@ -1634,9 +1634,9 @@ def test_custom_model_reuse_is_compatible( from raven.config.update_providers import set_provider_fields set_provider_fields("custom", {"api_key": "sk-cust", "api_base": "https://my-llm/v1"}) - assert onboard_commands._model_is_openai_compatible("qwen-max") + assert onboard_everos._model_is_openai_compatible("qwen-max") - creds = onboard_commands._resolve_reuse_llm_creds("qwen-max") + creds = onboard_everos._resolve_reuse_llm_creds("qwen-max") assert creds["model"] == "qwen-max" assert creds["api_key"] == "sk-cust" assert creds["base_url"] == "https://my-llm/v1" @@ -1854,7 +1854,7 @@ def _s3(**_): monkeypatch.setattr(onboard_commands, "_step1_provider", _s1) monkeypatch.setattr(onboard_commands, "_step2_sandbox", _s2) monkeypatch.setattr(onboard_commands, "_step3_channel", _s3) - monkeypatch.setattr(onboard_commands, "_step4_memory", lambda **_: None) + monkeypatch.setattr(onboard_everos, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) @@ -1883,7 +1883,7 @@ def test_first_screen_back_does_not_skip_step1( # Optional steps are no-ops here; we only assert Step 1 wasn't skipped. monkeypatch.setattr(onboard_commands, "_step2_sandbox", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step3_channel", lambda **_: None) - monkeypatch.setattr(onboard_commands, "_step4_memory", lambda **_: None) + monkeypatch.setattr(onboard_everos, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) @@ -1930,7 +1930,7 @@ def _verify(name, *a, **kw): monkeypatch.setattr(onboard_commands, "_failure_choice", lambda options, *, non_interactive: "switch") monkeypatch.setattr(onboard_commands, "_step2_sandbox", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step3_channel", lambda **_: None) - monkeypatch.setattr(onboard_commands, "_step4_memory", lambda **_: None) + monkeypatch.setattr(onboard_everos, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) @@ -1965,7 +1965,7 @@ def _fake_prompt_api_key(provider, **kw): monkeypatch.setattr(onboard_commands, "_pick_model", lambda provider, spec, **_: spec.default_model) monkeypatch.setattr(onboard_commands, "_step2_sandbox", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step3_channel", lambda **_: None) - monkeypatch.setattr(onboard_commands, "_step4_memory", lambda **_: None) + monkeypatch.setattr(onboard_everos, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) @@ -2440,7 +2440,7 @@ def _run_import_step( # whoever's machine runs the suite, not of the behaviour under test. Left # real, these tests pass on a developer box that has onboarded and fail # everywhere else, including CI. - monkeypatch.setattr(onboard_commands, "_memory_enabled", lambda: True) + monkeypatch.setattr(onboard_everos, "_memory_enabled", lambda: True) monkeypatch.setattr(onboard_commands, "_require_questionary", lambda: scripted) monkeypatch.setattr( "raven.importer.scanners.scan_all", @@ -2492,7 +2492,7 @@ def test_import_step_installs_skills_when_the_scan_finds_nothing( already covers, on the entry point that matters more. """ scripted = _ScriptedSelect([("import conversation history", "yes")]) - monkeypatch.setattr(onboard_commands, "_memory_enabled", lambda: True) + monkeypatch.setattr(onboard_everos, "_memory_enabled", lambda: True) monkeypatch.setattr(onboard_commands, "_require_questionary", lambda: scripted) monkeypatch.setattr("raven.importer.scanners.scan_all", AsyncMock(return_value=[])) _patch_skills_only_install(monkeypatch, tmp_path) @@ -2529,7 +2529,7 @@ def test_import_step_installs_skills_when_the_tier_keeps_nothing( ("Select import tier", Tier.MEMORY_FILES), ] ) - monkeypatch.setattr(onboard_commands, "_memory_enabled", lambda: True) + monkeypatch.setattr(onboard_everos, "_memory_enabled", lambda: True) monkeypatch.setattr(onboard_commands, "_require_questionary", lambda: scripted) monkeypatch.setattr("raven.importer.scanners.scan_all", AsyncMock(return_value=[conversation])) _patch_skill_count(monkeypatch, 12) @@ -2551,7 +2551,7 @@ def test_the_wizard_asks_before_copying_a_skill_tree( directory copy nothing undoes. """ scripted = _ScriptedSelect([("import conversation history", "yes")]) - monkeypatch.setattr(onboard_commands, "_memory_enabled", lambda: True) + monkeypatch.setattr(onboard_everos, "_memory_enabled", lambda: True) monkeypatch.setattr(onboard_commands, "_require_questionary", lambda: scripted) monkeypatch.setattr("raven.importer.scanners.scan_all", AsyncMock(return_value=[])) installer = _patch_skills_only_install(monkeypatch, tmp_path, confirm=False) @@ -3730,7 +3730,7 @@ def test_the_wizard_says_which_roles_everos_could_build( ) -> None: _stub_capabilities(monkeypatch, configured=("llm", "embedding"), llm=True, embed=True) - onboard_commands._report_everos_capabilities() + onboard_everos._report_everos_capabilities() assert "llm and embedding are available" in capsys.readouterr().out @@ -3743,7 +3743,7 @@ def test_the_wizard_flags_a_role_everos_could_not_build( degrades to keyword-only search, so a tick there would be a lie.""" _stub_capabilities(monkeypatch, configured=("llm", "embedding"), llm=True, embed=False) - onboard_commands._report_everos_capabilities() + onboard_everos._report_everos_capabilities() out = capsys.readouterr().out assert "embedding is configured but EverOS could not build it" in out @@ -3757,7 +3757,7 @@ def test_the_wizard_stays_quiet_on_a_server_that_cannot_report( condemn a working install.""" _stub_capabilities(monkeypatch, configured=("llm", "embedding")) - onboard_commands._report_everos_capabilities() + onboard_everos._report_everos_capabilities() assert capsys.readouterr().out.strip() == "" @@ -3769,7 +3769,7 @@ def test_the_llm_role_pre_fills_the_users_own_main_model() -> None: """A recommended model id is only a recommendation if the user's key can reach it, and many keys cannot. Their main model is one they demonstrably have, and the routing prefix has to come off for EverOS's bare client.""" - got = onboard_commands._preferred_memory_model("llm", "openrouter/anthropic/claude-sonnet-4-5", "openrouter") + got = onboard_everos._preferred_memory_model("llm", "openrouter/anthropic/claude-sonnet-4-5", "openrouter") assert got == "anthropic/claude-sonnet-4-5" @@ -3777,14 +3777,14 @@ def test_the_llm_role_pre_fills_the_users_own_main_model() -> None: def test_no_pre_fill_when_the_picked_provider_is_not_the_main_models() -> None: """No other provider carries that model id; pre-filling one it cannot serve would turn Enter into a verification failure.""" - got = onboard_commands._preferred_memory_model("llm", "openrouter/anthropic/claude-sonnet-4-5", "deepseek") + got = onboard_everos._preferred_memory_model("llm", "openrouter/anthropic/claude-sonnet-4-5", "deepseek") assert got is None def test_no_pre_fill_for_roles_that_do_not_serve_a_chat_model() -> None: for section in ("embedding", "rerank", "multimodal"): - got = onboard_commands._preferred_memory_model(section, "openrouter/anthropic/claude-sonnet-4-5", "openrouter") + got = onboard_everos._preferred_memory_model(section, "openrouter/anthropic/claude-sonnet-4-5", "openrouter") assert got is None, section @@ -3807,9 +3807,9 @@ def _autocomplete(_message, **kwargs): return _FQ() monkeypatch.setattr(questionary, "autocomplete", _autocomplete) - monkeypatch.setattr(onboard_commands, "_fetch_everos_models", lambda *a, **kw: ["a/b", "gpt-4.1-mini"]) + monkeypatch.setattr(onboard_everos, "_fetch_everos_models", lambda *a, **kw: ["a/b", "gpt-4.1-mini"]) - onboard_commands._everos_pick_model( + onboard_everos._everos_pick_model( base_url="https://x/v1", api_key="k", example="gpt-4.1-mini", @@ -3834,9 +3834,9 @@ def ask(self): return "chosen" monkeypatch.setattr(questionary, "autocomplete", lambda _m, **kw: (captured.update(kw), _FQ())[1]) - monkeypatch.setattr(onboard_commands, "_fetch_everos_models", lambda *a, **kw: ["x/gpt-4.1-mini"]) + monkeypatch.setattr(onboard_everos, "_fetch_everos_models", lambda *a, **kw: ["x/gpt-4.1-mini"]) - onboard_commands._everos_pick_model( + onboard_everos._everos_pick_model( base_url="https://x/v1", api_key="k", example="gpt-4.1-mini", @@ -3878,7 +3878,7 @@ def ask(self): return next(answers) monkeypatch.setattr(questionary, "select", lambda *a, **kw: _FQ()) - onboard_commands._step4_memory(skip=False, non_interactive=False, main_model="openai/gpt-4o-mini", warnings=[]) + onboard_everos._step4_memory(skip=False, non_interactive=False, main_model="openai/gpt-4o-mini", warnings=[]) out = " ".join(capsys.readouterr().out.split()) for needle in needles: @@ -3891,7 +3891,7 @@ def test_skipping_embedding_names_what_it_costs(monkeypatch: pytest.MonkeyPatch, altogether. The second cannot read like the first.""" monkeypatch.setattr(onboard_commands, "_LANG", lang) - note = onboard_commands._t(*onboard_commands._EVEROS_ROLES["embedding"]["skip_note"]) + note = onboard_commands._t(*onboard_everos._EVEROS_ROLES["embedding"]["skip_note"]) assert "yellow" in note, "a degradation this large must not be dim" assert "cascade backfill" in note @@ -3904,7 +3904,7 @@ def test_skipping_embedding_names_what_it_costs(monkeypatch: pytest.MonkeyPatch, def test_every_optional_role_carries_its_own_skip_note() -> None: """The renderer prints these verbatim now, so a note without its own markup would come out unstyled.""" - for name, role in onboard_commands._EVEROS_ROLES.items(): + for name, role in onboard_everos._EVEROS_ROLES.items(): if not role.get("optional"): continue note = role.get("skip_note") @@ -4068,7 +4068,7 @@ def _confirm(message: Any, **kw: Any) -> Any: def test_embedding_states_what_skipping_it_costs() -> None: """The one role whose absence changes how recall works at all -- searching lexically instead of semantically -- has to say so before it is skipped.""" - en, zh = onboard_commands._EVEROS_ROLES["embedding"]["cost"] + en, zh = onboard_everos._EVEROS_ROLES["embedding"]["cost"] assert "keywords" in en, en assert "关键词" in zh, zh @@ -4077,7 +4077,7 @@ def test_embedding_states_what_skipping_it_costs() -> None: def test_cost_lines_lead_with_the_consequence() -> None: """Whichever roles carry one, they read the same way, so a reader comparing two of them is comparing like with like.""" - for name, role in onboard_commands._EVEROS_ROLES.items(): + for name, role in onboard_everos._EVEROS_ROLES.items(): cost = role.get("cost") if not cost: continue @@ -4091,7 +4091,7 @@ def test_the_roles_we_want_configured_say_so(name: str) -> None: """Calling all three merely "optional" flattens the difference between losing semantic recall and losing some ranking accuracy. These two carry the encouragement in their own tag.""" - tag = onboard_commands._EVEROS_ROLES[name].get("tag") + tag = onboard_everos._EVEROS_ROLES[name].get("tag") assert tag, f"{name} should carry its own tag" en, zh = tag @@ -4106,7 +4106,7 @@ def test_role_blocks_fit_eighty_columns(monkeypatch: pytest.MonkeyPatch, lang: s from rich.text import Text monkeypatch.setattr(onboard_commands, "_LANG", lang) - for name, role in onboard_commands._EVEROS_ROLES.items(): + for name, role in onboard_everos._EVEROS_ROLES.items(): parts = [onboard_commands._t(*role["label"]), onboard_commands._t(*role["purpose"])] for key in ("tag", "cost", "recommendation", "skip_note"): if role.get(key): @@ -4140,7 +4140,7 @@ def ask(self): return "skip" monkeypatch.setattr(questionary, "select", lambda *a, **kw: _FQ()) - onboard_commands._config_everos_role( + onboard_everos._config_everos_role( section="embedding", main_model="openai/gpt-4o-mini", non_interactive=False, warnings=[] ) From dd81d0f914cb5c10112694e7505a53baaccdd8dd Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 00:29:04 +0800 Subject: [PATCH 21/78] fix(providers): only recover orphan think tags from parser-less backends split_orphan_think ran on every provider's every response, so any ordinary answer that mentioned a bare closing think tag -- a tutorial, a code sample -- had its leading half silently folded into reasoning. The parser-less sglang/vLLM shape only comes from a self-hosted backend, so emits_unparsed_reasoning gates both call sites: custom, local and unspecced identities normalize, gateways and known direct vendors leave content alone. The pairing check now looks for the shared opening-tag prefix, so a block that opens and closes no longer leaks its opener into the reasoning text. Co-authored-by: Claude (claude-fable-5) --- raven/agent/loop/main.py | 2 +- raven/providers/base.py | 9 +++++ raven/providers/litellm_provider.py | 22 ++++++++++- raven/providers/reasoning.py | 17 ++++++--- tests/test_agent_loop_stream.py | 42 ++++++++++++++++++--- tests/test_litellm_provider_attribution.py | 44 +++++++++++++++++++--- tests/test_provider_reasoning.py | 12 ++++++ 7 files changed, 130 insertions(+), 18 deletions(-) diff --git a/raven/agent/loop/main.py b/raven/agent/loop/main.py index 3892d4ff..f7a98666 100644 --- a/raven/agent/loop/main.py +++ b/raven/agent/loop/main.py @@ -1629,7 +1629,7 @@ async def _llm_call_stream( content = "".join(content_buf) reasoning_content = "".join(reasoning_buf) or None - if reasoning_content is None: + if reasoning_content is None and self.provider.emits_unparsed_reasoning(): split_reasoning, content = split_orphan_think(content) reasoning_content = split_reasoning diff --git a/raven/providers/base.py b/raven/providers/base.py index f8113bc3..64da5ca4 100644 --- a/raven/providers/base.py +++ b/raven/providers/base.py @@ -601,6 +601,15 @@ def can_serve(self, model: str) -> bool: """ return True + def emits_unparsed_reasoning(self) -> bool: + """Whether this provider's backend may leak bare think tags into content. + + Only an inference server run without its reasoning parser produces the + orphan-closing-tag shape; everyone else's `` in content is just + text. Default False: normalization is opt-in per provider shape. + """ + return False + @trace.instrument("llm.call", extract=semconv.llm_call) async def chat_with_retry( self, diff --git a/raven/providers/litellm_provider.py b/raven/providers/litellm_provider.py index 713a0169..dd3658c5 100644 --- a/raven/providers/litellm_provider.py +++ b/raven/providers/litellm_provider.py @@ -220,6 +220,26 @@ def can_serve(self, model: str) -> bool: return True return theirs.name == mine.name + def emits_unparsed_reasoning(self) -> bool: + """See ``LLMProvider.emits_unparsed_reasoning``. + + ``self._gateway``, when set, already answers this for both shapes it + can hold: a real network gateway (OpenRouter, AiHubMix) fronts one of + the large hosted vendors below it, so a bare ```` in content + is just content; the generic ``custom`` endpoint and a local spec + (hosted_vllm, ollama_chat) *are* the self-hosted inference server this + normalization exists for. When nothing was auto-detected, fall back to + whatever spec ``provider_name`` resolves to -- no spec at all (a bare + passthrough LiteLLM has no entry for) is the same self-hosted shape as + an explicit ``custom`` endpoint. Only a resolved spec that is neither + local nor ``custom`` -- a known direct big-vendor connection + (anthropic, openai, deepseek, ...) -- answers False: the + parser-less sglang/vLLM shape only comes from a self-hosted backend, + never from a vendor serving its own model behind its own API. + """ + spec = self._gateway or find_by_name(canonical_provider_name(self._provider_name)) + return spec is None or spec.is_local or spec.name == "custom" + def _supports_cache_control(self, model: str) -> bool: """Return True when this request may carry cache_control blocks. @@ -707,7 +727,7 @@ def _parse_response(self, response: Any) -> LLMResponse: reasoning_content = getattr(message, "reasoning_content", None) or None thinking_blocks = getattr(message, "thinking_blocks", None) or None - if not reasoning_content and isinstance(content, str): + if not reasoning_content and isinstance(content, str) and self.emits_unparsed_reasoning(): split_reasoning, content = split_orphan_think(content) reasoning_content = split_reasoning or reasoning_content diff --git a/raven/providers/reasoning.py b/raven/providers/reasoning.py index caf22636..62e1d947 100644 --- a/raven/providers/reasoning.py +++ b/raven/providers/reasoning.py @@ -43,10 +43,16 @@ def split_orphan_think(text: str) -> tuple[str | None, str]: An orphan is a ```` or ```` with no matching opening tag before it -- the shape produced when the server swallowed the opener into - its prompt template. A paired block (opener present earlier in the text) - is left alone and returned as ``(None, text)`` unchanged, for the existing - complete-block handlers (``_strip_think`` et al.) to take care of; so is - text with no closing tag at all. + its prompt template. A paired block (an opener present earlier in the + text) is left alone and returned as ``(None, text)`` unchanged, for the + existing complete-block handlers (``_strip_think`` et al.) to take care + of; so is text with no closing tag at all. + + The pairing check looks for the ```` and closes ```` (or vice versa) is still + a paired block, and treating the mismatched opener as absent would leak it + into the reasoning text this function returns. Everything before the tag becomes ``reasoning`` once stripped, unless that strips to nothing, in which case ``reasoning`` is ``None`` and only the @@ -58,9 +64,8 @@ def split_orphan_think(text: str) -> tuple[str | None, str]: if match is None: return None, text - open_tag = "" if match.group(0).lower() == "" else "" prefix = text[: match.start()] - if open_tag in prefix.lower(): + if " None: + ``emits_unparsed_reasoning`` defaults to False, mirroring + ``LLMProvider``'s own default: only a provider shaped like a parser-less + self-hosted backend opts into the orphan-```` split. + """ + + def __init__(self, chunks: list[StreamDelta], emits_unparsed_reasoning: bool = False) -> None: self._chunks = chunks self.chat_stream_calls: list[dict[str, Any]] = [] + self._emits_unparsed_reasoning = emits_unparsed_reasoning async def chat_stream(self, **kwargs: Any): self.chat_stream_calls.append(kwargs) for chunk in self._chunks: yield chunk + def emits_unparsed_reasoning(self) -> bool: + return self._emits_unparsed_reasoning + def _bind_helper(provider: _FakeProvider): """Bind ``_llm_call_stream`` to a SimpleNamespace stand-in for ``self``.""" @@ -299,7 +308,10 @@ async def on_delta(_text: str) -> None: # --------------------------------------------------------------------------- # Orphan recovery (issue #152) -- backend never emitted a structured # reasoning delta, and the accumulated content carries a closing tag with no -# opener (the server's prompt template swallowed it). +# opener (the server's prompt template swallowed it). Only fires for a +# provider shaped like a parser-less self-hosted backend +# (``emits_unparsed_reasoning() == True``); a normal direct/gateway provider +# leaves a bare closing tag in its content alone (F12). # --------------------------------------------------------------------------- @@ -309,7 +321,7 @@ async def test_llm_call_stream_splits_orphan_think_from_content() -> None: StreamDelta(content="\n"), StreamDelta(content="final answer"), ] - provider = _FakeProvider(chunks) + provider = _FakeProvider(chunks, emits_unparsed_reasoning=True) call = _bind_helper(provider) async def on_delta(_text: str) -> None: @@ -321,6 +333,26 @@ async def on_delta(_text: str) -> None: assert response.content == "final answer" +async def test_llm_call_stream_leaves_orphan_think_alone_for_non_leaking_provider() -> None: + """A provider not shaped like a parser-less self-hosted backend keeps a + bare closing tag as ordinary content (F12 regression guard).""" + chunks = [ + StreamDelta(content="discussing the "), + StreamDelta(content=""), + StreamDelta(content=" tag in my answer"), + ] + provider = _FakeProvider(chunks, emits_unparsed_reasoning=False) + call = _bind_helper(provider) + + async def on_delta(_text: str) -> None: + return None + + response = await call(messages=[], tools=None, model="m", on_token_delta=on_delta) + + assert response.reasoning_content is None + assert response.content == "discussing the tag in my answer" + + async def test_llm_call_stream_leaves_structured_reasoning_alone() -> None: """A non-empty structured reasoning_content stream wins outright; an orphan tag inside content (if any) is left untouched.""" @@ -328,7 +360,7 @@ async def test_llm_call_stream_leaves_structured_reasoning_alone() -> None: StreamDelta(content=None, reasoning_content="thinking"), StreamDelta(content="visible more text"), ] - provider = _FakeProvider(chunks) + provider = _FakeProvider(chunks, emits_unparsed_reasoning=True) call = _bind_helper(provider) async def on_delta(_text: str) -> None: diff --git a/tests/test_litellm_provider_attribution.py b/tests/test_litellm_provider_attribution.py index 2cca1841..b255e678 100644 --- a/tests/test_litellm_provider_attribution.py +++ b/tests/test_litellm_provider_attribution.py @@ -66,9 +66,13 @@ def test_extra_msg_keys_non_anthropic_preserves_nothing(): # --- orphan recovery in _parse_response (issue #152, keyless, no live call) --- # A backend run without a reasoning parser swallows the opening tag into its -# prompt template and returns bare reasoning text + a lone ``. Covers -# both directions: the split fires when there is no structured -# reasoning_content, and stays out of the way when there is one. +# prompt template and returns bare reasoning text + a lone ``. That +# shape only comes from a self-hosted inference server (hosted_vllm / custom / +# no spec at all) -- see `LiteLLMProvider.emits_unparsed_reasoning` -- so the +# split-fires cases below are built under one of those identities. A direct +# connection to a known hosted vendor (anthropic) or a real network gateway +# (openrouter) never gets normalized: a bare `` in their content is +# just content, not a leaked prompt template. def _fake_response(content: str, reasoning_content: str | None = None) -> MagicMock: @@ -78,7 +82,17 @@ def _fake_response(content: str, reasoning_content: str | None = None) -> MagicM def test_parse_response_splits_orphan_think_into_reasoning(): - provider = _make_provider("openai") + provider = _make_provider("hosted_vllm") + response = _fake_response("raw reasoning text\nfinal answer") + + result = provider._parse_response(response) + + assert result.reasoning_content == "raw reasoning text" + assert result.content == "final answer" + + +def test_parse_response_splits_orphan_think_for_custom_endpoint(): + provider = _make_provider("custom") response = _fake_response("raw reasoning text\nfinal answer") result = provider._parse_response(response) @@ -88,10 +102,30 @@ def test_parse_response_splits_orphan_think_into_reasoning(): def test_parse_response_leaves_structured_reasoning_content_alone(): - provider = _make_provider("openai") + provider = _make_provider("hosted_vllm") response = _fake_response("visible\nanswer", reasoning_content="already structured") result = provider._parse_response(response) assert result.reasoning_content == "already structured" assert result.content == "visible\nanswer" + + +def test_parse_response_leaves_bare_close_tag_alone_for_direct_anthropic(): + provider = _make_provider("anthropic") + response = _fake_response("discussing the tag in my answer") + + result = provider._parse_response(response) + + assert result.reasoning_content is None + assert result.content == "discussing the tag in my answer" + + +def test_parse_response_leaves_bare_close_tag_alone_behind_a_gateway(): + provider = _make_provider("openrouter") + response = _fake_response("discussing the tag in my answer") + + result = provider._parse_response(response) + + assert result.reasoning_content is None + assert result.content == "discussing the tag in my answer" diff --git a/tests/test_provider_reasoning.py b/tests/test_provider_reasoning.py index 89ee0c54..ad27534c 100644 --- a/tests/test_provider_reasoning.py +++ b/tests/test_provider_reasoning.py @@ -64,3 +64,15 @@ def test_content_without_a_leading_newline_is_unchanged(): assert reasoning == "reasoning" assert content == "immediate answer" + + +def test_mismatched_think_open_thinking_close_is_left_untouched(): + text = "raw reasoning\nfinal answer" + + assert split_orphan_think(text) == (None, text) + + +def test_mismatched_thinking_open_think_close_is_left_untouched(): + text = "raw reasoning\nfinal answer" + + assert split_orphan_think(text) == (None, text) From 7fd28a9fa270e7cec65b59631fa12d37e7196bf8 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 00:30:05 +0800 Subject: [PATCH 22/78] feat(providers): rotate and fail over across a provider's endpoints Second stage of multi-endpoint support: EndpointRotorProvider holds one inner provider per resolved endpoint and overrides _chat_attempt_with_retry, so the base class's model chain, cache-strip and can_serve guard all apply unchanged while each attempt walks the healthy endpoints -- sticky by default, round-robin by strategy, a failing endpoint cooling off 30s doubling to 300s, a non-fallback error returning immediately because another account cannot fix a bad request. Streams rotate only before the first real delta; once part of an answer exists, switching endpoints would duplicate or contradict it, so a later failure ends the stream like any single-endpoint provider's would. State is process-memory only, same reasoning as prompt_cache's suppression set. Co-authored-by: Claude (claude-fable-5) --- raven/providers/endpoint_rotor.py | 277 ++++++++++++++++++++++++++ tests/test_provider_endpoint_rotor.py | 269 +++++++++++++++++++++++++ 2 files changed, 546 insertions(+) create mode 100644 raven/providers/endpoint_rotor.py create mode 100644 tests/test_provider_endpoint_rotor.py diff --git a/raven/providers/endpoint_rotor.py b/raven/providers/endpoint_rotor.py new file mode 100644 index 00000000..dc03061d --- /dev/null +++ b/raven/providers/endpoint_rotor.py @@ -0,0 +1,277 @@ +"""Rotate and fail over across a provider section's several endpoints. + +``provider_endpoints`` (see ``raven.providers.endpoints``) resolves a section +into one or more ``ResolvedEndpoint``s -- several accounts on the same +vendor, several regions, several keys. This module is what #143/#144 asked +for on top of that list: spread requests across them (round-robin) or stick +to one until it misbehaves (sticky), and route around an endpoint that just +failed instead of sending the next request into the same wall. + +State (which endpoint is cooling, how many times it has failed, the rotation +cursor) lives in process memory only -- see ``RotorState`` for why, which is +the same reasoning as ``prompt_cache._SUPPRESSED``. + +A stream in progress is never switched mid-flight. Rotation only ever +happens before the caller has seen a token: an exception or an error-shaped +terminal delta arriving before the first real delta means nothing has been +said yet, so trying the next endpoint costs nothing. Once a normal first +delta has been handed to the caller, part of an answer already exists; +resuming it from a different endpoint would either duplicate or contradict +what was already sent, so a failure past that point is raised as-is and the +stream ends there, same as any single-endpoint provider's stream would. +""" + +from __future__ import annotations + +import time +from collections.abc import AsyncIterator, Callable +from contextlib import aclosing +from dataclasses import dataclass, field +from typing import Any + +from raven.providers.base import LLMProvider, LLMResponse, StreamDelta +from raven.providers.endpoints import ResolvedEndpoint + +#: Seconds a failed endpoint sits out before it is tried again, doubling per +#: consecutive failure (30, 60, 120, 240, 300-capped). Doubling lets a +#: momentary blip cost one skipped rotation while an endpoint that keeps +#: failing backs off further each time, rather than being retried on every +#: single request forever. +_COOLDOWN_INITIAL_SECONDS = 30.0 +_COOLDOWN_CAP_SECONDS = 300.0 + + +@dataclass +class RotorState: + """Per-instance rotation/failover bookkeeping -- process memory only. + + Not persisted, for the same reason as ``prompt_cache._SUPPRESSED``: which + endpoint is down right now is a live fact about this process's recent + calls, not a decision worth outliving it. The cost of forgetting on + restart is at most one bad request per endpoint before the next failure + re-learns it; the cost of a written file would be a healthy endpoint left + cooling because of a fact that stopped being true after the process that + wrote it exited. + + Concurrency: several ``chat``/``chat_stream`` calls can be in flight on + one instance, all on the same event loop. Every read and write here + happens between ``await`` points with no lock, which is safe under that + single-loop assumption -- there is no point where two of these methods run + interleaved. What *can* happen is two overlapping calls each hitting the + same endpoint and each independently calling ``mark_failure`` for it after + their own ``await`` returns; the second call just re-extends a cooldown + that was already in effect. That is redundant, not incorrect -- it never + leaves ``failure_count`` or ``cooldown_until`` in a state neither caller + intended. + """ + + index: int = 0 + cooldown_until: dict[int, float] = field(default_factory=dict) + failure_count: dict[int, int] = field(default_factory=dict) + + def is_cooling(self, i: int, now: float) -> bool: + return self.cooldown_until.get(i, 0.0) > now + + def mark_failure(self, i: int, now: float) -> None: + count = self.failure_count.get(i, 0) + 1 + self.failure_count[i] = count + cooldown = min(_COOLDOWN_CAP_SECONDS, _COOLDOWN_INITIAL_SECONDS * (2 ** (count - 1))) + self.cooldown_until[i] = now + cooldown + + def mark_success(self, i: int) -> None: + self.failure_count[i] = 0 + self.cooldown_until[i] = 0.0 + + +class EndpointRotorProvider(LLMProvider): + """Fan one provider section's several endpoints out behind one instance. + + ``make_inner`` builds the real per-endpoint provider (a real + ``LiteLLMProvider`` in production, a stub in tests) -- constructed eagerly + here, once, for every endpoint. Endpoints are already-resolved static + config (typically one to a handful), and building the inner provider does + no network I/O, so there is nothing to gain from deferring it and it keeps + ``_healthy_order()``'s index arithmetic pointed at a fixed, pre-built + list rather than re-invoking a factory per lookup. + """ + + def __init__( + self, + endpoints: list[ResolvedEndpoint], + make_inner: Callable[[ResolvedEndpoint], LLMProvider], + default_model: str, + strategy: str = "sticky", + ) -> None: + if not endpoints: + raise ValueError("EndpointRotorProvider requires at least one endpoint") + if strategy not in ("sticky", "round_robin"): + raise ValueError(f"unknown rotation strategy: {strategy!r}") + super().__init__() + self._endpoints = endpoints + self._inners = [make_inner(endpoint) for endpoint in endpoints] + self._default_model = default_model + self.strategy = strategy + self._state = RotorState() + + def _mark_failure(self, i: int) -> None: + self._state.mark_failure(i, time.monotonic()) + + def _mark_success(self, i: int) -> None: + self._state.mark_success(i) + + def _healthy_order(self) -> list[int]: + """Endpoint indices to try, in the order to try them. + + ``sticky`` always starts at index 0 and skips cooling entries -- one + endpoint stays "the" endpoint until it fails, matching the common + case of a single working account with spares for failover only. + ``round_robin`` starts at the rotor's cursor and advances the cursor + by one on every call, win or lose, so load spreads evenly across + several accounts rather than favoring whichever is first. + + When every endpoint is cooling, cooldown is ignored and the full + order is returned regardless of strategy -- a request has to go + somewhere, and cooldown is a preference between healthy endpoints, + not a breaker that can leave nothing to try. + """ + n = len(self._inners) + now = time.monotonic() + if self.strategy == "round_robin": + start = self._state.index + self._state.index = (start + 1) % n + order = [(start + i) % n for i in range(n)] + else: + order = list(range(n)) + healthy = [i for i in order if not self._state.is_cooling(i, now)] + return healthy or order + + async def _chat_attempt_with_retry( + self, + *, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]] | None, + model: str | None, + max_tokens: object, + temperature: object, + reasoning_effort: object, + tool_choice: str | dict[str, Any] | None, + ) -> LLMResponse: + """Run the retry ladder against each healthy endpoint in turn. + + Delegates to each endpoint's own inner ``_chat_attempt_with_retry`` -- + the retry ladder itself (backoff, prompt-cache-refusal downgrade) + stays that endpoint's job; this only decides which endpoint gets the + next attempt. A fallback-worthy exhaustion moves to the next endpoint + from ``_healthy_order()``; a non-fallback error (auth, + invalid_request, context overflow, ...) returns immediately, since a + different endpoint on the same account/vendor will not fix a rejected + key or a malformed request. Exhausting every endpoint returns the + last response, letting the caller's own model-chain fallback + (``LLMProvider.chat_with_retry``) take over from there. + """ + order = self._healthy_order() + last_response: LLMResponse | None = None + for i in order: + response = await self._inners[i]._chat_attempt_with_retry( + messages=messages, + tools=tools, + model=model, + max_tokens=max_tokens, + temperature=temperature, + reasoning_effort=reasoning_effort, + tool_choice=tool_choice, + ) + if response.finish_reason != "error": + self._mark_success(i) + return response + + classification = response.error_classification or self.classify_error(content=response.content) + response.error_classification = classification + last_response = response + if not classification.should_fallback: + return response + self._mark_failure(i) + + return last_response # type: ignore[return-value] # order always non-empty + + async def chat( + self, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]] | None = None, + model: str | None = None, + **kwargs: Any, + ) -> LLMResponse: + """Single-shot call to the first healthy endpoint -- see ``_healthy_order``. + + No in-call rotation: a caller reaching ``chat`` directly (bypassing + ``chat_with_retry``) gets exactly one endpoint's answer, same as any + other provider's ``chat``. Rotating on failure is the retry layer's + job, handled by ``_chat_attempt_with_retry``. + """ + idx = self._healthy_order()[0] + return await self._inners[idx].chat(messages, tools, model=model, **kwargs) + + async def chat_stream( + self, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]] | None = None, + model: str | None = None, + **kwargs: Any, + ) -> AsyncIterator[StreamDelta]: + """Open-stream endpoint rotation -- see the module docstring for why + rotation stops the moment a real delta has been yielded. + + ``aclosing`` guarantees a stream abandoned mid-rotation (an endpoint + that failed before its first delta) is closed before the next one is + opened, mirroring ``LiteLLMProvider.chat_stream``'s own close-on-every- + exit discipline for the underlying HTTP stream. + """ + order = self._healthy_order() + last_failure: Exception | StreamDelta | None = None + for i in order: + inner = self._inners[i] + async with aclosing(inner.chat_stream(messages, tools, model=model, **kwargs)) as agen: + try: + first = await agen.__anext__() + except StopAsyncIteration: + self._mark_success(i) + return + except Exception as exc: + classification = self.classify_error(exc) + if not classification.should_fallback: + raise + self._mark_failure(i) + last_failure = exc + continue + + if first.finish_reason == "error": + # The fallback path 34099d8 added surfaces a failed open as + # a terminal error delta rather than an exception; judged + # the same way as one. + classification = first.error_classification or self.classify_error(content=first.content) + if classification.should_fallback: + self._mark_failure(i) + last_failure = first + continue + yield first + return + + self._mark_success(i) + yield first + async for delta in agen: + yield delta + return + + if isinstance(last_failure, Exception): + raise last_failure + if last_failure is not None: + yield last_failure + + def can_serve(self, model: str) -> bool: + """Delegates to the first endpoint's inner -- every endpoint under one + rotor is the same vendor/section, so their identity for routing + purposes is one answer, not one per endpoint.""" + return self._inners[0].can_serve(model) + + def get_default_model(self) -> str: + return self._default_model diff --git a/tests/test_provider_endpoint_rotor.py b/tests/test_provider_endpoint_rotor.py new file mode 100644 index 00000000..a19075cb --- /dev/null +++ b/tests/test_provider_endpoint_rotor.py @@ -0,0 +1,269 @@ +"""Rotation and failover across a provider section's several endpoints. + +Covers: +- sticky: first endpoint stays "the" endpoint until it fails, cooldown + transfers to the next, and expiry restores it +- round_robin: the cursor advances by one on every call +- a non-fallback error (auth) never rotates +- every endpoint cooling still dispatches (no deadlock) +- chat_stream: transfer before the first delta, no transfer after it, + no token replay +- _chat_attempt_with_retry composes with the base class's own model-chain + fallback (chat_with_retry) +- cooldown doubles per failure, capped, and clears on success +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from raven.providers import endpoint_rotor +from raven.providers.base import ErrorClassification, LLMProvider, LLMResponse, StreamDelta +from raven.providers.endpoint_rotor import EndpointRotorProvider, RotorState +from raven.providers.endpoints import ResolvedEndpoint + + +class _StubInner(LLMProvider): + """Records calls; ``chat()`` pops one scripted response per call and + ``chat_stream()`` pops one scripted list of deltas/exceptions per call. + + An empty queue defaults to a plain "ok" response/stream so tests that + don't care about a given endpoint's exact reply don't need to script it. + """ + + def __init__(self, name: str, chat_script: list[Any] | None = None, stream_script: list[list[Any]] | None = None): + super().__init__(api_key="test") + self.name = name + self._chat_script = list(chat_script or []) + self._stream_script = list(stream_script or []) + self.chat_calls = 0 + self.stream_calls = 0 + self._CHAT_RETRY_DELAYS = (0, 0, 0) + + async def chat(self, messages, tools=None, model=None, **kwargs) -> LLMResponse: + self.chat_calls += 1 + if self._chat_script: + return self._chat_script.pop(0) + return LLMResponse(content=f"ok:{self.name}", finish_reason="stop") + + async def chat_stream(self, messages, tools=None, model=None, **kwargs): + self.stream_calls += 1 + script = self._stream_script.pop(0) if self._stream_script else [StreamDelta(content=f"ok:{self.name}")] + for item in script: + if isinstance(item, BaseException): + raise item + yield item + + def get_default_model(self) -> str: + return self.name + + +def _endpoint(label: str) -> ResolvedEndpoint: + return ResolvedEndpoint(label=label, api_key=f"key-{label}", api_base=None, extra_headers=None) + + +def _make_rotor(inners: list[_StubInner], strategy: str = "sticky") -> EndpointRotorProvider: + endpoints = [_endpoint(inner.name) for inner in inners] + by_label = {e.label: inner for e, inner in zip(endpoints, inners)} + return EndpointRotorProvider( + endpoints=endpoints, + make_inner=lambda e: by_label[e.label], + default_model="rotor-default", + strategy=strategy, + ) + + +_FALLBACK_FATAL = ErrorClassification(category="model_unavailable", should_fallback=True) +_NON_FALLBACK_FATAL = ErrorClassification(category="auth") + + +class _Clock: + """Monotonic stand-in a test can advance explicitly.""" + + def __init__(self, start: float = 0.0): + self.now = start + + def __call__(self) -> float: + return self.now + + +@pytest.fixture +def clock(monkeypatch): + c = _Clock() + monkeypatch.setattr(endpoint_rotor.time, "monotonic", c) + return c + + +async def test_sticky_cools_on_fallback_error_and_recovers_after_expiry(clock): + e0 = _StubInner("e0", chat_script=[LLMResponse(content="ok0", finish_reason="stop")]) + e1 = _StubInner("e1", chat_script=[LLMResponse(content="ok1", finish_reason="stop")]) + rotor = _make_rotor([e0, e1], strategy="sticky") + + resp = await rotor.chat_with_retry(messages=[], model="m") + assert resp.content == "ok0" + assert (e0.chat_calls, e1.chat_calls) == (1, 0) + + e0._chat_script.append( + LLMResponse(content="e0 unavailable", finish_reason="error", error_classification=_FALLBACK_FATAL) + ) + resp = await rotor.chat_with_retry(messages=[], model="m") + assert resp.content == "ok1" + assert (e0.chat_calls, e1.chat_calls) == (2, 1) + + # e0 is cooling: sticky skips it and goes straight to e1 again, no new e0 attempt. + e1._chat_script.append(LLMResponse(content="ok1 again", finish_reason="stop")) + resp = await rotor.chat_with_retry(messages=[], model="m") + assert resp.content == "ok1 again" + assert (e0.chat_calls, e1.chat_calls) == (2, 2) + + # Cooldown (30s for a first failure) expires: sticky returns to e0. + clock.now += 30.0 + e0._chat_script.append(LLMResponse(content="ok0 again", finish_reason="stop")) + resp = await rotor.chat_with_retry(messages=[], model="m") + assert resp.content == "ok0 again" + assert (e0.chat_calls, e1.chat_calls) == (3, 2) + + +async def test_round_robin_cursor_advances_each_call(clock): + e0 = _StubInner("e0") + e1 = _StubInner("e1") + e2 = _StubInner("e2") + rotor = _make_rotor([e0, e1, e2], strategy="round_robin") + + order_seen = [] + for _ in range(4): + resp = await rotor.chat_with_retry(messages=[], model="m") + order_seen.append(resp.content) + + # Cursor starts at 0 and advances by one on every call, wrapping at 3. + assert order_seen == ["ok:e0", "ok:e1", "ok:e2", "ok:e0"] + assert (e0.chat_calls, e1.chat_calls, e2.chat_calls) == (2, 1, 1) + + +async def test_non_fallback_error_returns_immediately_without_rotating(clock): + e0 = _StubInner( + "e0", + chat_script=[ + LLMResponse(content="401 unauthorized", finish_reason="error", error_classification=_NON_FALLBACK_FATAL) + ], + ) + e1 = _StubInner("e1") + rotor = _make_rotor([e0, e1], strategy="sticky") + + resp = await rotor.chat_with_retry(messages=[], model="m") + + assert resp.finish_reason == "error" + assert resp.content == "401 unauthorized" + assert (e0.chat_calls, e1.chat_calls) == (1, 0) + # A fatal-but-not-fallback-worthy error must not cool the endpoint either -- + # there was nothing wrong with the endpoint, the request was invalid. + assert rotor._state.is_cooling(0, clock.now) is False + + +async def test_all_endpoints_cooling_still_dispatches_in_order(clock): + e0 = _StubInner("e0", chat_script=[LLMResponse(content="ok0", finish_reason="stop")]) + e1 = _StubInner("e1") + rotor = _make_rotor([e0, e1], strategy="sticky") + + rotor._state.mark_failure(0, clock.now) + rotor._state.mark_failure(1, clock.now) + assert rotor._healthy_order() == [0, 1] + + resp = await rotor.chat_with_retry(messages=[], model="m") + + assert resp.content == "ok0" + assert e0.chat_calls == 1 + + +async def test_stream_transfers_on_open_exception_before_any_delta(clock): + e0 = _StubInner("e0", stream_script=[[RuntimeError("503 service unavailable")]]) + e1 = _StubInner("e1", stream_script=[[StreamDelta(content="hi"), StreamDelta(content=" world")]]) + rotor = _make_rotor([e0, e1], strategy="sticky") + + seen = [d.content async for d in rotor.chat_stream(messages=[])] + + assert seen == ["hi", " world"] + assert (e0.stream_calls, e1.stream_calls) == (1, 1) + assert rotor._state.is_cooling(0, clock.now) is True + + +async def test_stream_transfers_on_error_terminal_first_delta(clock): + error_delta = StreamDelta(content=None, finish_reason="error", error_classification=_FALLBACK_FATAL) + e0 = _StubInner("e0", stream_script=[[error_delta]]) + e1 = _StubInner("e1", stream_script=[[StreamDelta(content="ok")]]) + rotor = _make_rotor([e0, e1], strategy="sticky") + + seen = [d.content async for d in rotor.chat_stream(messages=[])] + + assert seen == ["ok"] + assert (e0.stream_calls, e1.stream_calls) == (1, 1) + assert rotor._state.is_cooling(0, clock.now) is True + + +async def test_stream_does_not_transfer_after_a_normal_first_delta(clock): + e0 = _StubInner("e0", stream_script=[[StreamDelta(content="a"), RuntimeError("mid-stream boom")]]) + e1 = _StubInner("e1", stream_script=[[StreamDelta(content="should-not-be-used")]]) + rotor = _make_rotor([e0, e1], strategy="sticky") + + seen = [] + with pytest.raises(RuntimeError, match="mid-stream boom"): + async for d in rotor.chat_stream(messages=[]): + seen.append(d.content) + + # The token before the failure was delivered exactly once, not replayed + # from a second endpoint, and e1 was never even tried. + assert seen == ["a"] + assert e1.stream_calls == 0 + + +async def test_chat_attempt_with_retry_composes_with_base_model_chain_fallback(clock): + """Both endpoints exhaust on model A (should_fallback) -> the base + class's own chat_with_retry moves to model B -> the rotor answers B from + whichever endpoint is tried first (both are cooling by then, so sticky + ignores cooldown and starts back at e0).""" + e0 = _StubInner( + "e0", + chat_script=[ + LLMResponse(content="a unavailable on e0", finish_reason="error", error_classification=_FALLBACK_FATAL), + LLMResponse(content="recovered", finish_reason="stop"), + ], + ) + e1 = _StubInner( + "e1", + chat_script=[ + LLMResponse(content="a unavailable on e1", finish_reason="error", error_classification=_FALLBACK_FATAL), + ], + ) + rotor = _make_rotor([e0, e1], strategy="sticky") + + resp = await rotor.chat_with_retry(messages=[], model="A", fallback_models=["B"]) + + assert resp.content == "recovered" + assert (e0.chat_calls, e1.chat_calls) == (2, 1) + + +async def test_cooldown_doubles_per_failure_capped_and_clears_on_success(clock): + state = RotorState() + + state.mark_failure(0, clock.now) + assert state.cooldown_until[0] == pytest.approx(30.0) + + clock.now = 30.0 + state.mark_failure(0, clock.now) + assert state.cooldown_until[0] == pytest.approx(30.0 + 60.0) + + clock.now = 90.0 + state.mark_failure(0, clock.now) + assert state.cooldown_until[0] == pytest.approx(90.0 + 120.0) + + # Keep failing until the doubling would exceed the cap; it must clamp. + clock.now = 500.0 + state.failure_count[0] = 10 # next doubled value (30 * 2**10) is far past the cap + state.mark_failure(0, clock.now) + assert state.cooldown_until[0] == pytest.approx(500.0 + 300.0) + + state.mark_success(0) + assert state.failure_count[0] == 0 + assert state.is_cooling(0, clock.now) is False From 73a3906ae0edef40329de63f717d551b83eec8f8 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 00:32:34 +0800 Subject: [PATCH 23/78] refactor(*): keep pin wording out of the window ladder terms Provider Pin is a registered domain term (agents.defaults.provider); the window ladder's construction-time flag borrowed "pinned" for an unrelated concept, and the Token Rates entry still said an unknown window falls back to the caller's configured default, which the ladder made untrue. The flag and its tests now say "explicit", and the entry describes the ladder and the gauge's honest empty state. Co-authored-by: Claude (claude-fable-5) --- CONTEXT.md | 7 +++++-- raven/agent/loop/main.py | 32 ++++++++++++++++------------- tests/test_agent_loop_usage_sink.py | 24 +++++++++++----------- 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 47d3b03f..4afc98e5 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -271,8 +271,11 @@ about a Provider's catalogue, so both are resolved in `providers/rates.py` rathe whoever is about to report a number. The two are deliberately sourced differently: rates price a call after it happened, so the ladder may reach a community-maintained catalogue; a context window sizes trimming and therefore shapes the *next* request, so only the -tables that also route may answer it, and unknown is answered with the caller's own -configured default. +tables that also route may answer it. The window walks its own ladder +(`effective_context_window`): an explicitly configured value wins outright, then the +model's real window, then the module's documented fallback -- and a gauge that cannot +resolve the real window reports 0 so the UI shows its empty state rather than a number +that is nobody's. _Avoid_: "pricing" for the resolution -- that names the arithmetic on top (`token_wise/pricing.py`), which is a different module for a reason. diff --git a/raven/agent/loop/main.py b/raven/agent/loop/main.py index f7a98666..ff5a5d26 100644 --- a/raven/agent/loop/main.py +++ b/raven/agent/loop/main.py @@ -378,9 +378,11 @@ def __init__( self.max_iterations = max_iterations # Empty-response recovery budgets. None → enabled defaults. self._recovery_limits = empty_recovery if empty_recovery is not None else RecoveryLimits() - # A caller that passed a positive value pinned the window; None/0 means - # "figure it out", resolved once here against the model's real window. - self._context_window_pinned = bool(context_window_tokens) + # A caller that passed a positive value set the window explicitly; + # None/0 means "figure it out", resolved once here against the model's + # real window. ("Explicit", not "pinned" -- Provider Pin is a different + # registered term, see CONTEXT.md.) + self._context_window_explicit = bool(context_window_tokens) # allow_fetch=False: construction must not block on a synchronous # network call for an OpenRouter model's window -- whatever is already # cached (in-process or on disk, any age) answers instead. See @@ -706,12 +708,13 @@ def configure_personalization(self, enable: bool) -> None: def refresh_context_window(self) -> None: """Re-resolve ``context_window_tokens`` against the current ``self.model``. - A no-op once the window was pinned at construction -- a pin is a - deliberate override, and a model switch afterwards must not quietly - discard it. Unpinned loops re-walk the ladder so a ``/model`` switch - picks up the new model's real window instead of keeping the old one's. + A no-op once the window was set explicitly at construction -- an + explicit value is a deliberate override, and a model switch afterwards + must not quietly discard it. Otherwise the ladder is re-walked so a + ``/model`` switch picks up the new model's real window instead of + keeping the old one's. """ - if self._context_window_pinned: + if self._context_window_explicit: return # allow_fetch=False: a /model switch runs inside the running event # loop, so this must not block it on a synchronous network call. See @@ -1940,12 +1943,13 @@ async def _run_agent_loop( if usage_sink is not None and response.usage: prompt_tokens = int(response.usage.get("prompt_tokens", 0) or 0) completion_tokens = int(response.usage.get("completion_tokens", 0) or 0) - # A pin always wins over the live table -- that is what pinning - # means. Unpinned, the live window from the model's provider - # table (e.g. OpenRouter, when LiteLLM lags) answers instead; - # unknown to that table too, 0 tells the UI to show its empty - # state rather than a number that isn't this model's. - if self._context_window_pinned: + # An explicitly configured window always wins over the live + # table -- that is what setting it means. Otherwise the live + # window from the model's provider table (e.g. OpenRouter, + # when LiteLLM lags) answers instead; unknown to that table + # too, 0 tells the UI to show its empty state rather than a + # number that isn't this model's. + if self._context_window_explicit: context_max = self.context_window_tokens else: context_max = resolve_context_window(call_model) or 0 diff --git a/tests/test_agent_loop_usage_sink.py b/tests/test_agent_loop_usage_sink.py index 2d5e3ac1..c9690bd9 100644 --- a/tests/test_agent_loop_usage_sink.py +++ b/tests/test_agent_loop_usage_sink.py @@ -130,7 +130,7 @@ def client_factory(*args, **kwargs): @pytest.mark.asyncio async def test_usage_sink_context_max_from_live_openrouter(workspace, monkeypatch): - """Unpinned, an OpenRouter model LiteLLM lags on gets its real window from /models.""" + """Without an explicit window, an OpenRouter model LiteLLM lags on gets its real window from /models.""" _patch_live_openrouter_window(monkeypatch, 163840) provider = UsageProvider("openrouter/deepseek/deepseek-v4-pro", 1000, 500) @@ -157,8 +157,8 @@ async def test_usage_sink_context_max_from_live_openrouter(workspace, monkeypatc @pytest.mark.asyncio -async def test_usage_sink_context_max_stays_pinned_over_live_openrouter(workspace, monkeypatch): - """A pinned window wins even when the model's live window disagrees.""" +async def test_usage_sink_context_max_stays_explicit_over_live_openrouter(workspace, monkeypatch): + """An explicitly configured window wins even when the model's live window disagrees.""" _patch_live_openrouter_window(monkeypatch, 163840) provider = UsageProvider("openrouter/deepseek/deepseek-v4-pro", 1000, 500) @@ -185,28 +185,28 @@ async def test_usage_sink_context_max_stays_pinned_over_live_openrouter(workspac # --------------------------------------------------------------------------- # -# construction-time ladder: _context_window_pinned + refresh_context_window # +# construction-time ladder: _context_window_explicit + refresh_context_window # # --------------------------------------------------------------------------- # -def test_no_configured_window_resolves_via_the_ladder_and_is_unpinned(workspace): - """An unresolvable model falls back to the documented default, unpinned.""" +def test_no_configured_window_resolves_via_the_ladder_and_is_not_explicit(workspace): + """An unresolvable model falls back to the documented default, not explicit.""" provider = UsageProvider("stub", 0, 0) agent = _make_agent(workspace, provider, model="stub", window=None) - assert agent._context_window_pinned is False + assert agent._context_window_explicit is False assert agent.context_window_tokens == rates.DEFAULT_CONTEXT_WINDOW_TOKENS -def test_a_configured_window_is_pinned_at_construction(workspace): +def test_a_configured_window_is_explicit_at_construction(workspace): provider = UsageProvider("stub", 0, 0) agent = _make_agent(workspace, provider, model="stub", window=8192) - assert agent._context_window_pinned is True + assert agent._context_window_explicit is True assert agent.context_window_tokens == 8192 -def test_refresh_context_window_is_a_noop_once_pinned(workspace, monkeypatch): +def test_refresh_context_window_is_a_noop_once_explicit(workspace, monkeypatch): """A pin is a deliberate override; a later model switch must not discard it.""" _patch_live_openrouter_window(monkeypatch, 163840) @@ -219,8 +219,8 @@ def test_refresh_context_window_is_a_noop_once_pinned(workspace, monkeypatch): assert agent.context_window_tokens == 8192 -def test_refresh_context_window_follows_the_new_model_when_unpinned(workspace, monkeypatch): - """Unpinned, a ``/model`` switch re-walks the ladder for the new model. +def test_refresh_context_window_follows_the_new_model_when_not_explicit(workspace, monkeypatch): + """Without an explicit window, a ``/model`` switch re-walks the ladder for the new model. The switch runs inside the running event loop, so the ladder is walked with ``allow_fetch=False`` -- an in-process cache entry of any age answers rather From bdf7aaa6244c8d290320cc590c3cc53da2b8af95 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 00:36:44 +0800 Subject: [PATCH 24/78] refactor(cli): move the wizard's channel cluster into its own module Second cut of the onboard split: the Step3 chat-channel cluster (~580 lines, 15 symbols) moves to onboard_channels.py under the same discipline as the everos cut -- shared wizard state stays in onboard_commands, both sides reach across through module references with call-time attribute access, and every migrated patch target was mutation-checked one by one (12/12 red against the old module). Also renames _step5_import to _step6_import: its own step header already said 6, and the misnamed function sat next to the real _step5 (deep research), waiting to misdirect the next split. onboard_commands.py is now 3000 lines, down from 5069. Co-authored-by: Claude (claude-fable-5) --- raven/cli/onboard_channels.py | 582 +++++++++++++++++++++++++++++ raven/cli/onboard_commands.py | 574 +--------------------------- tests/test_cli_onboard_commands.py | 64 ++-- 3 files changed, 619 insertions(+), 601 deletions(-) create mode 100644 raven/cli/onboard_channels.py diff --git a/raven/cli/onboard_channels.py b/raven/cli/onboard_channels.py new file mode 100644 index 00000000..81df3153 --- /dev/null +++ b/raven/cli/onboard_channels.py @@ -0,0 +1,582 @@ +"""Chat channel cluster of the onboard wizard (Step 3). + +Split out of ``onboard_commands`` because that module had grown past 5000 +lines; this file owns channel selection, credential prompting, scancode +login, and channel management end to end. Shared wizard UI state +(``console``, ``_t``, ``_BACK``, ``_QMARK``, questionary helpers, ...) still +lives in ``onboard_commands`` -- this module reaches it via the ``oc`` module +reference (not a value import) so that test monkeypatches on +``onboard_commands`` attributes keep working whichever module a caller +patches through. +""" + +from __future__ import annotations + +from typing import Any, Optional + +import typer + +from raven.cli import onboard_commands as oc + + +def _enabled_channels() -> list[str]: + """Names of channels currently enabled on disk.""" + data = oc._load_raw_config() + channels = data.get("channels") or {} + return [name for name, c in channels.items() if isinstance(c, dict) and c.get("enabled")] + + +# Curated channel order: China-domestic first, then overseas. Channels not +# listed (e.g. a newly added adapter) fall to the end in alphabetical order so +# the picker never silently hides one. +# Display order: US/global-common → China-common → US/global-uncommon → +# China-uncommon. (Email is a universal but less-common-as-IM channel, so it +# sits in the uncommon tail.) +_CHANNEL_ORDER = ( + # US / global, common + "telegram", + "discord", + "slack", + "whatsapp", + # China, common + "weixin", + "wecom", + "feishu", + "dingtalk", + "qq", + # US / global, less common + "matrix", + "email", + # China, niche + "mochat", +) + + +# Where to obtain each channel's credentials — shown (dim) before the field +# prompts so the user knows where to fetch the token / keys. +_CHANNEL_CRED_HELP: dict[str, tuple[str, str]] = { + "telegram": ( + "Create a bot with @BotFather in Telegram (send /newbot) — it replies with the token.", + "在 Telegram 里找 @BotFather 发 /newbot 创建机器人,它会回复 token。", + ), + "discord": ( + "Discord Developer Portal → your app → Bot → Reset Token to copy it.", + "Discord 开发者门户 → 你的应用 → Bot → Reset Token 复制。", + ), + "slack": ( + "api.slack.com/apps → OAuth & Permissions gives bot_token (xoxb-…); " + "Basic Information → App-Level Tokens gives app_token (xapp-…).", + "api.slack.com/apps → OAuth & Permissions 拿 bot_token(xoxb-…);" + "Basic Information → App-Level Tokens 拿 app_token(xapp-…)。", + ), + "feishu": ( + "Feishu / Lark Open Platform → your app → Credentials for App ID & App Secret.", + "飞书开放平台 → 你的应用 → 凭证与基础信息 拿 App ID / App Secret。", + ), + "wecom": ( + "WeCom admin console → your bot / app for its ID and secret.", + "企业微信管理后台 → 机器人 / 应用 拿 ID 和 secret。", + ), + "dingtalk": ( + "DingTalk Open Platform → your app for Client ID & Client Secret.", + "钉钉开放平台 → 你的应用 拿 Client ID / Client Secret。", + ), + "qq": ( + "QQ Open Platform → your bot for App ID & secret.", + "QQ 开放平台 → 你的机器人 拿 App ID 和 secret。", + ), + "email": ( + "Use your mail provider's IMAP / SMTP settings; for Gmail / Outlook create an app password.", + "用你邮箱服务商的 IMAP / SMTP 设置;Gmail / Outlook 需创建应用专用密码。", + ), + "matrix": ( + "From your Matrix account: an access token and your full user id (@you:server).", + "从你的 Matrix 账号获取 access token 和完整用户 id(@you:server)。", + ), + "mochat": ( + "Get the claw token and agent user id from your Mochat workspace.", + "从你的 Mochat 工作区获取 claw token 和 agent user id。", + ), +} + + +def _ordered_channel_names() -> list[str]: + from raven.channels.registry import discover_channel_names + + rank = {name: i for i, name in enumerate(_CHANNEL_ORDER)} + return sorted(discover_channel_names(), key=lambda n: (rank.get(n, len(rank)), n)) + + +def _select_channel() -> Optional[str]: + """List available channels via the registry and let the user pick one.""" + questionary = oc._require_questionary() + from raven.cli._styles import RAVEN_STYLE + + names = _ordered_channel_names() + choices = [questionary.Choice(n, value=n) for n in names] + choices.append(questionary.Choice(oc._t("Back", "返回"), value=oc._BACK)) + picked = questionary.select(oc._t("Channel:", "渠道:"), choices=choices, style=RAVEN_STYLE, qmark=oc._QMARK).ask() + return picked + + +def _prompt_channel_fields(channel: str) -> Any: + """Reflect a channel's Pydantic schema and prompt for credential-like fields.""" + questionary = oc._require_questionary() + from raven.cli._styles import RAVEN_STYLE + from raven.config.update_channels import channel_field_specs + + try: + specs = channel_field_specs(channel) + except KeyError as exc: + oc.console.print(f" [red]✗[/red] {exc}") + raise typer.Exit(1) + + # Pre-scan which credential fields we'll ask for, so we can tell the user + # up front what's being configured (and handle the zero-field case). + promptable = [ + (path, spec) + for path, spec in specs.items() + if path != "enabled" and spec.get("type", "") == "str" and spec.get("default") in ("", None) + ] + if promptable: + names = ", ".join(path for path, _ in promptable) + oc.console.print( + oc._t( + f" [dim]Configuring {channel} — fill in:[/dim] {names}", + f" [dim]正在配置 {channel} — 请填写:[/dim] {names}", + ) + ) + help_text = _CHANNEL_CRED_HELP.get(channel) + if help_text: + oc.console.print( + oc._t( + f" [dim]Where to get it: {help_text[0]}[/dim]", + f" [dim]去哪拿:{help_text[1]}[/dim]", + ) + ) + else: + oc.console.print( + oc._t( + f" [dim]{channel} needs no credentials; enabling.[/dim]", + f" [dim]{channel} 无需填写凭证,正在启用。[/dim]", + ) + ) + + fields: dict[str, Any] = {} + for idx, (path, spec) in enumerate(promptable): + required = bool(spec.get("required")) + description = spec.get("description", "") + opt_tag = "" if required else oc._t(" (optional)", " (可选)") + prompt_label = f"{path}{opt_tag}" + (f" — {description}" if description else "") + ":" + # First field's empty submit rewinds to the channel picker; a later + # optional field's empty submit skips it; a later required field re-prompts + # (empty was previously accepted silently, enabling a half-configured + # channel — the write layer treats "required" as a UX marker only). + allow_back = idx == 0 + placeholder = oc._field_placeholder(allow_back, required) + while True: + if spec.get("is_secret"): + value = questionary.password( + prompt_label, placeholder=placeholder, style=RAVEN_STYLE, qmark=oc._QMARK + ).ask() + else: + value = questionary.text( + prompt_label, placeholder=placeholder, style=RAVEN_STYLE, qmark=oc._QMARK + ).ask() + if value is None: + raise typer.Exit(1) + value = value.strip() + if value: + fields[path] = value + break + if allow_back: + return oc._BACK # first field empty → back to the channel picker + if required: + oc.console.print( + oc._t(f" [yellow]{path} is required.[/yellow]", f" [yellow]{path} 为必填项。[/yellow]") + ) + continue # re-prompt instead of enabling a channel missing a credential + break # optional field: empty submit skips it + return fields + + +def _enable_channel(channel: str, fields: dict[str, Any]) -> None: + """Thin wrapper for ``enable_channel`` that surfaces ops errors with hints.""" + from pydantic import ValidationError + + from raven.config.update_channels import enable_channel + + try: + enable_channel(channel, fields) + except KeyError as exc: + oc.console.print(f" [red]✗[/red] {exc}") + raise typer.Exit(1) + except ValidationError as exc: + oc.console.print(oc._t(f" [red]✗ Validation failed:[/red]\n{exc}", f" [red]✗ 校验失败:[/red]\n{exc}")) + raise typer.Exit(1) + + +def _channel_uses_interactive_login(channel: str) -> bool: + """True for scancode/QR channels (WeChat / WhatsApp) that pair via a live + login flow rather than reflected credential fields.""" + try: + from raven.channels.registry import discover_specs + + spec = discover_specs().get(channel) + return bool(spec and spec.capabilities.interactive_login) + except Exception: + return False + + +# Scancode channels whose QR login is served by a Node.js bridge — these need +# Node/npm present before login can even start. The whatsapp adapter's +# ``login`` checks ``shutil.which("npm")`` and merely logs+returns False when +# it's absent, so we detect the missing-runtime case up front to show a +# meaningful "install Node / skip" menu rather than a pointless "re-show QR". +_NODE_BRIDGE_CHANNELS = {"whatsapp"} + + +def _node_runtime_missing(channel: str) -> bool: + """True iff ``channel`` needs a Node bridge and ``npm`` isn't on PATH.""" + if channel not in _NODE_BRIDGE_CHANNELS: + return False + import shutil + + return shutil.which("npm") is None + + +def _handle_missing_node(channel: str, *, non_interactive: bool) -> str: + """Show the Node-missing submenu (install-then-retry / skip). + + Returns ``"retry"`` (re-check after install) or ``"skip"`` (leave the + channel enabled-but-unauthenticated). A pointless "re-show QR" is + intentionally absent — there's no bridge to render a QR without Node. + """ + oc.console.print( + oc._t( + f" [yellow]✗ Node.js / npm not found (the {channel} bridge needs it). " + "Install Node.js, then retry.[/yellow]", + f" [yellow]✗ 未找到 Node.js / npm({channel} 的桥接需要它)。请先安装 Node.js,再重试。[/yellow]", + ) + ) + choice = oc._failure_choice( + [ + (oc._t("Retry after install", "安装后重试"), "retry"), + (oc._t("Skip", "跳过"), "skip"), + ], + non_interactive=non_interactive, + ) + return choice + + +def _scancode_login(channel: str, *, non_interactive: bool = False) -> None: + """Run a scancode channel's real QR login (reuses ``channel.login``). + + Mirrors ``raven channels login``: enable the channel so its config section + persists, build the adapter via its spec factory, then drive + ``await channel.login()`` (which for WhatsApp builds the bridge, displays + the QR, and waits). A failed / timed-out login drops into a numbered + submenu (retry / skip). Node-bridge channels missing Node/npm get a + dedicated install-then-retry menu instead. + """ + import asyncio + + from raven.channels.registry import discover_specs + from raven.config.update_channels import disable_channel + + # Enable first so the config section exists for the factory to read while we + # attempt login. We REVERT this (disable) on any path that doesn't complete + # login, so a cancelled / skipped scan never shows up as "connected". + _enable_channel(channel, {}) + + specs = discover_specs() + spec = specs.get(channel) + if spec is None: + disable_channel(channel) + oc.console.print(oc._t(f" [red]✗ Unknown channel: {channel}[/red]", f" [red]✗ 未知渠道:{channel}[/red]")) + return + + # Enabled above so the factory can read the config section during login. ANY + # path that doesn't finish login must revert the enable — including Ctrl+C in + # a submenu (raises typer.Exit) or mid-scan (KeyboardInterrupt), neither an + # ``Exception`` subclass — so wrap the whole flow and disable in ``finally`` + # unless we actually logged in. + logged_in = False + try: + while True: + # Node-bridge channels: gate on the runtime up front so a missing + # Node/npm shows a useful install menu, not a "re-show QR" no-op. + if _node_runtime_missing(channel): + if _handle_missing_node(channel, non_interactive=non_interactive) == "retry": + continue + oc.console.print( + oc._t( + f" [dim]Skipped {channel}; install Node.js then run raven channels login {channel}.[/dim]", + f" [dim]已跳过 {channel};装好 Node.js 后运行 raven channels login {channel}。[/dim]", + ) + ) + return + + from raven.config.loader import load_config + + channel_cfg = getattr(load_config().channels, channel, None) + if channel_cfg is None: + oc.console.print( + oc._t( + f" [red]✗ No config section for channel: {channel}[/red]", + f" [red]✗ 渠道 {channel} 没有配置段。[/red]", + ) + ) + return + adapter = spec.factory(channel_cfg) + if channel == "whatsapp": + oc.console.print( + oc._t( + " [dim]Building the WhatsApp bridge — the first run can take 30–120s…[/dim]", + " [dim]正在构建 WhatsApp 桥接,首次约需 30–120 秒…[/dim]", + ) + ) + oc.console.print( + oc._t( + f" [dim]Starting {spec.display_name} QR login…[/dim]", + f" [dim]正在启动 {spec.display_name} 扫码登录…[/dim]", + ) + ) + oc.console.print( + oc._t( + f" [dim]A login link / QR code will appear below — scan it with " + f"{spec.display_name} (or open the link on a phone signed in to " + f"{spec.display_name}) to connect. This waits until you finish.[/dim]", + f" [dim]下方会出现登录链接 / 二维码 — 用 {spec.display_name} 扫码" + f"(或在已登录 {spec.display_name} 的手机上打开该链接)即可接入;" + f"这里会一直等到你完成。[/dim]", + ) + ) + from loguru import logger as _wiz_logger + + # The wizard silences raven logs for a clean UI, but a scancode login + # emits its QR / link / progress / failure reason through loguru. Re- + # enable ONLY this channel's adapter subtree for the login attempt (not + # all of raven, which would dump unrelated noise), then restore quiet. + _login_log_scope = f"raven.channels.adapters.{channel}" + try: + _wiz_logger.enable(_login_log_scope) + ok = asyncio.run(adapter.login(force=True)) + except Exception as exc: + oc.console.print( + oc._t( + f" [yellow]✗ Login failed: {exc}[/yellow]", + f" [yellow]✗ 登录失败:{exc}[/yellow]", + ) + ) + ok = False + finally: + _wiz_logger.disable(_login_log_scope) + if ok: + oc.console.print( + oc._t( + f" [green]✓ Logged in; {channel} connected.[/green]", + f" [green]✓ 已登录;{channel} 已接入。[/green]", + ) + ) + logged_in = True + return + choice = oc._failure_choice( + [ + (oc._t("Retry", "重试"), "retry"), + (oc._t("Skip this channel", "跳过此渠道"), "skip"), + ], + non_interactive=non_interactive, + ) + if choice == "retry": + continue + oc.console.print( + oc._t( + f" [dim]{channel} not connected — finish later with raven channels login {channel}.[/dim]", + f" [dim]{channel} 未接入 — 之后用 raven channels login {channel} 完成。[/dim]", + ) + ) + return + finally: + if not logged_in: + # Any non-login exit (skip, no-config, submenu Ctrl+C, mid-scan + # interrupt) reverts the enable so a cancelled scan never persists as + # "connected". The config section is kept for `raven channels login`. + disable_channel(channel) + + +def _add_one_channel(*, non_interactive: bool = False) -> None: + """Pick + (scancode login | reflect-prompt) + enable one channel.""" + while True: + channel = _select_channel() + if channel is None or channel is oc._BACK: + return + if _channel_uses_interactive_login(channel): + _scancode_login(channel, non_interactive=non_interactive) + return + fields = _prompt_channel_fields(channel) + if fields is oc._BACK: + continue # backed out of the first field — re-pick a channel + _enable_channel(channel, fields) + oc.console.print(oc._t(f" [green]✓ {channel} enabled.[/green]", f" [green]✓ {channel} 已启用。[/green]")) + return + + +def _manage_existing_channels() -> None: + """Edit/disable submenu for already-enabled channels.""" + questionary = oc._require_questionary() + from raven.cli._styles import RAVEN_STYLE + from raven.config.update_channels import disable_channel, set_channel_fields + + while True: + enabled = _enabled_channels() + if not enabled: + return + choices = [questionary.Choice(n, value=n) for n in enabled] + choices.append(questionary.Choice(oc._t("Back", "返回"), value=oc._BACK)) + target = questionary.select( + oc._t("Pick a channel to manage:", "选择要管理的渠道:"), + choices=choices, + style=RAVEN_STYLE, + qmark=oc._QMARK, + ).ask() + if target is None or target is oc._BACK: + return + action = questionary.select( + oc._t(f"What would you like to do with {target}?", f"对 {target} 想做什么?"), + choices=[ + questionary.Choice(oc._t("Edit config (re-enter fields)", "编辑配置(重填字段)"), value="edit"), + questionary.Choice(oc._t("Disable (keep credentials)", "停用(保留凭证)"), value="disable"), + questionary.Choice(oc._t("Back", "返回"), value=oc._BACK), + ], + style=RAVEN_STYLE, + qmark=oc._QMARK, + ).ask() + if action is None or action is oc._BACK: + continue + if action == "edit": + fields = _prompt_channel_fields(target) + if fields is oc._BACK: + continue # backed out — return to the manage menu + if fields: + set_channel_fields(target, fields) + oc.console.print( + oc._t( + f" [green]✓ {target} config updated.[/green]", + f" [green]✓ {target} 配置已更新。[/green]", + ) + ) + elif action == "disable": + disable_channel(target) + oc.console.print( + oc._t( + f" [green]✓ Disabled {target} (credentials kept; re-enable later " + f"with raven channels enable {target}).[/green]", + f" [green]✓ 已停用 {target}(凭证保留;之后用 raven channels enable {target} 重新启用)。[/green]", + ) + ) + + +def _step3_channel(*, channel: Optional[str], skip: bool, non_interactive: bool) -> object: + """Step 3 — optionally enable chat channel(s).""" + oc._step_header( + 3, + oc._t( + "(Optional) Connect a messaging app so you can chat with Raven there", + "(可选)接入即时通讯软件,直接在里面和 Raven 聊天", + ), + ) + + if skip: + oc.console.print( + oc._t( + " [dim]Skipped via --skip-channel.[/dim]", + " [dim]已通过 --skip-channel 跳过。[/dim]", + ) + ) + return None + + if non_interactive: + if channel: + oc.console.print( + f"[red]--channel {channel} given but non-interactive mode can't " + "prompt for credential fields.[/red]\n" + f"Run [accent]raven channels enable {channel} -- ...[/accent] " + "after onboard finishes." + ) + raise typer.Exit(2) + oc.console.print( + oc._t( + " [dim]Skipped (non-interactive, --channel not given).[/dim]", + " [dim]已跳过(非交互且未提供 --channel)。[/dim]", + ) + ) + return None + + questionary = oc._require_questionary() + from raven.cli._styles import RAVEN_STYLE + + if channel: + if _channel_uses_interactive_login(channel): + _scancode_login(channel, non_interactive=non_interactive) + else: + fields = _prompt_channel_fields(channel) + if fields is oc._BACK: + oc.console.print(oc._t(" [dim]Skipped.[/dim]", " [dim]已跳过。[/dim]")) + return None + _enable_channel(channel, fields) + oc.console.print( + oc._t( + f" [green]✓ {channel} enabled.[/green]", + f" [green]✓ {channel} 已启用。[/green]", + ) + ) + return None + + while True: + enabled = _enabled_channels() + if not enabled: + action = questionary.select( + oc._t("Connect a chat channel?", "接入一个聊天渠道吗?"), + choices=[ + questionary.Choice(oc._t("Add a channel", "新增一个渠道"), value="add"), + questionary.Choice( + oc._t( + "Skip (add later with raven channels enable)", + "跳过(之后用 raven channels enable 添加)", + ), + value="skip", + ), + ], + style=RAVEN_STYLE, + qmark=oc._QMARK, + ).ask() + if action is None: + raise typer.Exit(1) + if action == "skip": + oc.console.print(oc._t(" [dim]Skipped.[/dim]", " [dim]已跳过。[/dim]")) + return None + _add_one_channel(non_interactive=non_interactive) + continue + + action = questionary.select( + oc._t( + f"Chat channel already connected: {', '.join(enabled)}. What would you like to do?", + f"聊天渠道已接入:{', '.join(enabled)}。想做什么?", + ), + choices=[ + questionary.Choice(oc._t("Done, next step", "完成,下一步"), value="done"), + questionary.Choice(oc._t("Add a channel", "新增一个渠道"), value="add"), + questionary.Choice(oc._t("Edit / remove a channel", "编辑 / 移除渠道"), value="edit"), + ], + style=RAVEN_STYLE, + qmark=oc._QMARK, + ).ask() + if action is None: + raise typer.Exit(1) + if action == "done": + return None + if action == "add": + _add_one_channel(non_interactive=non_interactive) + elif action == "edit": + _manage_existing_channels() diff --git a/raven/cli/onboard_commands.py b/raven/cli/onboard_commands.py index 4701929c..9a50ee5c 100644 --- a/raven/cli/onboard_commands.py +++ b/raven/cli/onboard_commands.py @@ -36,7 +36,7 @@ from rich.console import Console from rich.panel import Panel -from raven.cli import onboard_everos +from raven.cli import onboard_channels, onboard_everos from raven.cli._helpers import ( DEFAULT_PROBE_MESSAGE, print_probe_troubleshooting, @@ -2128,570 +2128,6 @@ def _step2_sandbox(*, skip: bool, non_interactive: bool) -> object: return None -# --------------------------------------------------------------------------- -# Step 3 — chat channel (stackable) -# --------------------------------------------------------------------------- - - -def _enabled_channels() -> list[str]: - """Names of channels currently enabled on disk.""" - data = _load_raw_config() - channels = data.get("channels") or {} - return [name for name, c in channels.items() if isinstance(c, dict) and c.get("enabled")] - - -# Curated channel order: China-domestic first, then overseas. Channels not -# listed (e.g. a newly added adapter) fall to the end in alphabetical order so -# the picker never silently hides one. -# Display order: US/global-common → China-common → US/global-uncommon → -# China-uncommon. (Email is a universal but less-common-as-IM channel, so it -# sits in the uncommon tail.) -_CHANNEL_ORDER = ( - # US / global, common - "telegram", - "discord", - "slack", - "whatsapp", - # China, common - "weixin", - "wecom", - "feishu", - "dingtalk", - "qq", - # US / global, less common - "matrix", - "email", - # China, niche - "mochat", -) - - -# Where to obtain each channel's credentials — shown (dim) before the field -# prompts so the user knows where to fetch the token / keys. -_CHANNEL_CRED_HELP: dict[str, tuple[str, str]] = { - "telegram": ( - "Create a bot with @BotFather in Telegram (send /newbot) — it replies with the token.", - "在 Telegram 里找 @BotFather 发 /newbot 创建机器人,它会回复 token。", - ), - "discord": ( - "Discord Developer Portal → your app → Bot → Reset Token to copy it.", - "Discord 开发者门户 → 你的应用 → Bot → Reset Token 复制。", - ), - "slack": ( - "api.slack.com/apps → OAuth & Permissions gives bot_token (xoxb-…); " - "Basic Information → App-Level Tokens gives app_token (xapp-…).", - "api.slack.com/apps → OAuth & Permissions 拿 bot_token(xoxb-…);" - "Basic Information → App-Level Tokens 拿 app_token(xapp-…)。", - ), - "feishu": ( - "Feishu / Lark Open Platform → your app → Credentials for App ID & App Secret.", - "飞书开放平台 → 你的应用 → 凭证与基础信息 拿 App ID / App Secret。", - ), - "wecom": ( - "WeCom admin console → your bot / app for its ID and secret.", - "企业微信管理后台 → 机器人 / 应用 拿 ID 和 secret。", - ), - "dingtalk": ( - "DingTalk Open Platform → your app for Client ID & Client Secret.", - "钉钉开放平台 → 你的应用 拿 Client ID / Client Secret。", - ), - "qq": ( - "QQ Open Platform → your bot for App ID & secret.", - "QQ 开放平台 → 你的机器人 拿 App ID 和 secret。", - ), - "email": ( - "Use your mail provider's IMAP / SMTP settings; for Gmail / Outlook create an app password.", - "用你邮箱服务商的 IMAP / SMTP 设置;Gmail / Outlook 需创建应用专用密码。", - ), - "matrix": ( - "From your Matrix account: an access token and your full user id (@you:server).", - "从你的 Matrix 账号获取 access token 和完整用户 id(@you:server)。", - ), - "mochat": ( - "Get the claw token and agent user id from your Mochat workspace.", - "从你的 Mochat 工作区获取 claw token 和 agent user id。", - ), -} - - -def _ordered_channel_names() -> list[str]: - from raven.channels.registry import discover_channel_names - - rank = {name: i for i, name in enumerate(_CHANNEL_ORDER)} - return sorted(discover_channel_names(), key=lambda n: (rank.get(n, len(rank)), n)) - - -def _select_channel() -> Optional[str]: - """List available channels via the registry and let the user pick one.""" - questionary = _require_questionary() - from raven.cli._styles import RAVEN_STYLE - - names = _ordered_channel_names() - choices = [questionary.Choice(n, value=n) for n in names] - choices.append(questionary.Choice(_t("Back", "返回"), value=_BACK)) - picked = questionary.select(_t("Channel:", "渠道:"), choices=choices, style=RAVEN_STYLE, qmark=_QMARK).ask() - return picked - - -def _prompt_channel_fields(channel: str) -> Any: - """Reflect a channel's Pydantic schema and prompt for credential-like fields.""" - questionary = _require_questionary() - from raven.cli._styles import RAVEN_STYLE - from raven.config.update_channels import channel_field_specs - - try: - specs = channel_field_specs(channel) - except KeyError as exc: - console.print(f" [red]✗[/red] {exc}") - raise typer.Exit(1) - - # Pre-scan which credential fields we'll ask for, so we can tell the user - # up front what's being configured (and handle the zero-field case). - promptable = [ - (path, spec) - for path, spec in specs.items() - if path != "enabled" and spec.get("type", "") == "str" and spec.get("default") in ("", None) - ] - if promptable: - names = ", ".join(path for path, _ in promptable) - console.print( - _t( - f" [dim]Configuring {channel} — fill in:[/dim] {names}", - f" [dim]正在配置 {channel} — 请填写:[/dim] {names}", - ) - ) - help_text = _CHANNEL_CRED_HELP.get(channel) - if help_text: - console.print( - _t( - f" [dim]Where to get it: {help_text[0]}[/dim]", - f" [dim]去哪拿:{help_text[1]}[/dim]", - ) - ) - else: - console.print( - _t( - f" [dim]{channel} needs no credentials; enabling.[/dim]", - f" [dim]{channel} 无需填写凭证,正在启用。[/dim]", - ) - ) - - fields: dict[str, Any] = {} - for idx, (path, spec) in enumerate(promptable): - required = bool(spec.get("required")) - description = spec.get("description", "") - opt_tag = "" if required else _t(" (optional)", " (可选)") - prompt_label = f"{path}{opt_tag}" + (f" — {description}" if description else "") + ":" - # First field's empty submit rewinds to the channel picker; a later - # optional field's empty submit skips it; a later required field re-prompts - # (empty was previously accepted silently, enabling a half-configured - # channel — the write layer treats "required" as a UX marker only). - allow_back = idx == 0 - placeholder = _field_placeholder(allow_back, required) - while True: - if spec.get("is_secret"): - value = questionary.password( - prompt_label, placeholder=placeholder, style=RAVEN_STYLE, qmark=_QMARK - ).ask() - else: - value = questionary.text(prompt_label, placeholder=placeholder, style=RAVEN_STYLE, qmark=_QMARK).ask() - if value is None: - raise typer.Exit(1) - value = value.strip() - if value: - fields[path] = value - break - if allow_back: - return _BACK # first field empty → back to the channel picker - if required: - console.print(_t(f" [yellow]{path} is required.[/yellow]", f" [yellow]{path} 为必填项。[/yellow]")) - continue # re-prompt instead of enabling a channel missing a credential - break # optional field: empty submit skips it - return fields - - -def _enable_channel(channel: str, fields: dict[str, Any]) -> None: - """Thin wrapper for ``enable_channel`` that surfaces ops errors with hints.""" - from pydantic import ValidationError - - from raven.config.update_channels import enable_channel - - try: - enable_channel(channel, fields) - except KeyError as exc: - console.print(f" [red]✗[/red] {exc}") - raise typer.Exit(1) - except ValidationError as exc: - console.print(_t(f" [red]✗ Validation failed:[/red]\n{exc}", f" [red]✗ 校验失败:[/red]\n{exc}")) - raise typer.Exit(1) - - -def _channel_uses_interactive_login(channel: str) -> bool: - """True for scancode/QR channels (WeChat / WhatsApp) that pair via a live - login flow rather than reflected credential fields.""" - try: - from raven.channels.registry import discover_specs - - spec = discover_specs().get(channel) - return bool(spec and spec.capabilities.interactive_login) - except Exception: - return False - - -# Scancode channels whose QR login is served by a Node.js bridge — these need -# Node/npm present before login can even start. The whatsapp adapter's -# ``login`` checks ``shutil.which("npm")`` and merely logs+returns False when -# it's absent, so we detect the missing-runtime case up front to show a -# meaningful "install Node / skip" menu rather than a pointless "re-show QR". -_NODE_BRIDGE_CHANNELS = {"whatsapp"} - - -def _node_runtime_missing(channel: str) -> bool: - """True iff ``channel`` needs a Node bridge and ``npm`` isn't on PATH.""" - if channel not in _NODE_BRIDGE_CHANNELS: - return False - import shutil - - return shutil.which("npm") is None - - -def _handle_missing_node(channel: str, *, non_interactive: bool) -> str: - """Show the Node-missing submenu (install-then-retry / skip). - - Returns ``"retry"`` (re-check after install) or ``"skip"`` (leave the - channel enabled-but-unauthenticated). A pointless "re-show QR" is - intentionally absent — there's no bridge to render a QR without Node. - """ - console.print( - _t( - f" [yellow]✗ Node.js / npm not found (the {channel} bridge needs it). " - "Install Node.js, then retry.[/yellow]", - f" [yellow]✗ 未找到 Node.js / npm({channel} 的桥接需要它)。请先安装 Node.js,再重试。[/yellow]", - ) - ) - choice = _failure_choice( - [ - (_t("Retry after install", "安装后重试"), "retry"), - (_t("Skip", "跳过"), "skip"), - ], - non_interactive=non_interactive, - ) - return choice - - -def _scancode_login(channel: str, *, non_interactive: bool = False) -> None: - """Run a scancode channel's real QR login (reuses ``channel.login``). - - Mirrors ``raven channels login``: enable the channel so its config section - persists, build the adapter via its spec factory, then drive - ``await channel.login()`` (which for WhatsApp builds the bridge, displays - the QR, and waits). A failed / timed-out login drops into a numbered - submenu (retry / skip). Node-bridge channels missing Node/npm get a - dedicated install-then-retry menu instead. - """ - import asyncio - - from raven.channels.registry import discover_specs - from raven.config.update_channels import disable_channel - - # Enable first so the config section exists for the factory to read while we - # attempt login. We REVERT this (disable) on any path that doesn't complete - # login, so a cancelled / skipped scan never shows up as "connected". - _enable_channel(channel, {}) - - specs = discover_specs() - spec = specs.get(channel) - if spec is None: - disable_channel(channel) - console.print(_t(f" [red]✗ Unknown channel: {channel}[/red]", f" [red]✗ 未知渠道:{channel}[/red]")) - return - - # Enabled above so the factory can read the config section during login. ANY - # path that doesn't finish login must revert the enable — including Ctrl+C in - # a submenu (raises typer.Exit) or mid-scan (KeyboardInterrupt), neither an - # ``Exception`` subclass — so wrap the whole flow and disable in ``finally`` - # unless we actually logged in. - logged_in = False - try: - while True: - # Node-bridge channels: gate on the runtime up front so a missing - # Node/npm shows a useful install menu, not a "re-show QR" no-op. - if _node_runtime_missing(channel): - if _handle_missing_node(channel, non_interactive=non_interactive) == "retry": - continue - console.print( - _t( - f" [dim]Skipped {channel}; install Node.js then run raven channels login {channel}.[/dim]", - f" [dim]已跳过 {channel};装好 Node.js 后运行 raven channels login {channel}。[/dim]", - ) - ) - return - - from raven.config.loader import load_config - - channel_cfg = getattr(load_config().channels, channel, None) - if channel_cfg is None: - console.print( - _t( - f" [red]✗ No config section for channel: {channel}[/red]", - f" [red]✗ 渠道 {channel} 没有配置段。[/red]", - ) - ) - return - adapter = spec.factory(channel_cfg) - if channel == "whatsapp": - console.print( - _t( - " [dim]Building the WhatsApp bridge — the first run can take 30–120s…[/dim]", - " [dim]正在构建 WhatsApp 桥接,首次约需 30–120 秒…[/dim]", - ) - ) - console.print( - _t( - f" [dim]Starting {spec.display_name} QR login…[/dim]", - f" [dim]正在启动 {spec.display_name} 扫码登录…[/dim]", - ) - ) - console.print( - _t( - f" [dim]A login link / QR code will appear below — scan it with " - f"{spec.display_name} (or open the link on a phone signed in to " - f"{spec.display_name}) to connect. This waits until you finish.[/dim]", - f" [dim]下方会出现登录链接 / 二维码 — 用 {spec.display_name} 扫码" - f"(或在已登录 {spec.display_name} 的手机上打开该链接)即可接入;" - f"这里会一直等到你完成。[/dim]", - ) - ) - from loguru import logger as _wiz_logger - - # The wizard silences raven logs for a clean UI, but a scancode login - # emits its QR / link / progress / failure reason through loguru. Re- - # enable ONLY this channel's adapter subtree for the login attempt (not - # all of raven, which would dump unrelated noise), then restore quiet. - _login_log_scope = f"raven.channels.adapters.{channel}" - try: - _wiz_logger.enable(_login_log_scope) - ok = asyncio.run(adapter.login(force=True)) - except Exception as exc: - console.print( - _t( - f" [yellow]✗ Login failed: {exc}[/yellow]", - f" [yellow]✗ 登录失败:{exc}[/yellow]", - ) - ) - ok = False - finally: - _wiz_logger.disable(_login_log_scope) - if ok: - console.print( - _t( - f" [green]✓ Logged in; {channel} connected.[/green]", - f" [green]✓ 已登录;{channel} 已接入。[/green]", - ) - ) - logged_in = True - return - choice = _failure_choice( - [ - (_t("Retry", "重试"), "retry"), - (_t("Skip this channel", "跳过此渠道"), "skip"), - ], - non_interactive=non_interactive, - ) - if choice == "retry": - continue - console.print( - _t( - f" [dim]{channel} not connected — finish later with raven channels login {channel}.[/dim]", - f" [dim]{channel} 未接入 — 之后用 raven channels login {channel} 完成。[/dim]", - ) - ) - return - finally: - if not logged_in: - # Any non-login exit (skip, no-config, submenu Ctrl+C, mid-scan - # interrupt) reverts the enable so a cancelled scan never persists as - # "connected". The config section is kept for `raven channels login`. - disable_channel(channel) - - -def _add_one_channel(*, non_interactive: bool = False) -> None: - """Pick + (scancode login | reflect-prompt) + enable one channel.""" - while True: - channel = _select_channel() - if channel is None or channel is _BACK: - return - if _channel_uses_interactive_login(channel): - _scancode_login(channel, non_interactive=non_interactive) - return - fields = _prompt_channel_fields(channel) - if fields is _BACK: - continue # backed out of the first field — re-pick a channel - _enable_channel(channel, fields) - console.print(_t(f" [green]✓ {channel} enabled.[/green]", f" [green]✓ {channel} 已启用。[/green]")) - return - - -def _manage_existing_channels() -> None: - """Edit/disable submenu for already-enabled channels.""" - questionary = _require_questionary() - from raven.cli._styles import RAVEN_STYLE - from raven.config.update_channels import disable_channel, set_channel_fields - - while True: - enabled = _enabled_channels() - if not enabled: - return - choices = [questionary.Choice(n, value=n) for n in enabled] - choices.append(questionary.Choice(_t("Back", "返回"), value=_BACK)) - target = questionary.select( - _t("Pick a channel to manage:", "选择要管理的渠道:"), - choices=choices, - style=RAVEN_STYLE, - qmark=_QMARK, - ).ask() - if target is None or target is _BACK: - return - action = questionary.select( - _t(f"What would you like to do with {target}?", f"对 {target} 想做什么?"), - choices=[ - questionary.Choice(_t("Edit config (re-enter fields)", "编辑配置(重填字段)"), value="edit"), - questionary.Choice(_t("Disable (keep credentials)", "停用(保留凭证)"), value="disable"), - questionary.Choice(_t("Back", "返回"), value=_BACK), - ], - style=RAVEN_STYLE, - qmark=_QMARK, - ).ask() - if action is None or action is _BACK: - continue - if action == "edit": - fields = _prompt_channel_fields(target) - if fields is _BACK: - continue # backed out — return to the manage menu - if fields: - set_channel_fields(target, fields) - console.print( - _t( - f" [green]✓ {target} config updated.[/green]", - f" [green]✓ {target} 配置已更新。[/green]", - ) - ) - elif action == "disable": - disable_channel(target) - console.print( - _t( - f" [green]✓ Disabled {target} (credentials kept; re-enable later " - f"with raven channels enable {target}).[/green]", - f" [green]✓ 已停用 {target}(凭证保留;之后用 raven channels enable {target} 重新启用)。[/green]", - ) - ) - - -def _step3_channel(*, channel: Optional[str], skip: bool, non_interactive: bool) -> object: - """Step 3 — optionally enable chat channel(s).""" - _step_header( - 3, - _t( - "(Optional) Connect a messaging app so you can chat with Raven there", - "(可选)接入即时通讯软件,直接在里面和 Raven 聊天", - ), - ) - - if skip: - console.print( - _t( - " [dim]Skipped via --skip-channel.[/dim]", - " [dim]已通过 --skip-channel 跳过。[/dim]", - ) - ) - return None - - if non_interactive: - if channel: - console.print( - f"[red]--channel {channel} given but non-interactive mode can't " - "prompt for credential fields.[/red]\n" - f"Run [accent]raven channels enable {channel} -- ...[/accent] " - "after onboard finishes." - ) - raise typer.Exit(2) - console.print( - _t( - " [dim]Skipped (non-interactive, --channel not given).[/dim]", - " [dim]已跳过(非交互且未提供 --channel)。[/dim]", - ) - ) - return None - - questionary = _require_questionary() - from raven.cli._styles import RAVEN_STYLE - - if channel: - if _channel_uses_interactive_login(channel): - _scancode_login(channel, non_interactive=non_interactive) - else: - fields = _prompt_channel_fields(channel) - if fields is _BACK: - console.print(_t(" [dim]Skipped.[/dim]", " [dim]已跳过。[/dim]")) - return None - _enable_channel(channel, fields) - console.print( - _t( - f" [green]✓ {channel} enabled.[/green]", - f" [green]✓ {channel} 已启用。[/green]", - ) - ) - return None - - while True: - enabled = _enabled_channels() - if not enabled: - action = questionary.select( - _t("Connect a chat channel?", "接入一个聊天渠道吗?"), - choices=[ - questionary.Choice(_t("Add a channel", "新增一个渠道"), value="add"), - questionary.Choice( - _t( - "Skip (add later with raven channels enable)", - "跳过(之后用 raven channels enable 添加)", - ), - value="skip", - ), - ], - style=RAVEN_STYLE, - qmark=_QMARK, - ).ask() - if action is None: - raise typer.Exit(1) - if action == "skip": - console.print(_t(" [dim]Skipped.[/dim]", " [dim]已跳过。[/dim]")) - return None - _add_one_channel(non_interactive=non_interactive) - continue - - action = questionary.select( - _t( - f"Chat channel already connected: {', '.join(enabled)}. What would you like to do?", - f"聊天渠道已接入:{', '.join(enabled)}。想做什么?", - ), - choices=[ - questionary.Choice(_t("Done, next step", "完成,下一步"), value="done"), - questionary.Choice(_t("Add a channel", "新增一个渠道"), value="add"), - questionary.Choice(_t("Edit / remove a channel", "编辑 / 移除渠道"), value="edit"), - ], - style=RAVEN_STYLE, - qmark=_QMARK, - ).ask() - if action is None: - raise typer.Exit(1) - if action == "done": - return None - if action == "add": - _add_one_channel(non_interactive=non_interactive) - elif action == "edit": - _manage_existing_channels() - - # --------------------------------------------------------------------------- # Final summary # --------------------------------------------------------------------------- @@ -2742,7 +2178,7 @@ def _print_next_steps(*, warnings: list[str]) -> None: if _current_sandbox_backend() == "none" else _t("Sandbox (boxlite)", "沙箱(boxlite)") ) - chans = ", ".join(_enabled_channels()) or _t("none", "无") + chans = ", ".join(onboard_channels._enabled_channels()) or _t("none", "无") mem = ( _t("EverOS", "EverOS") if onboard_everos._memory_enabled() @@ -2811,7 +2247,7 @@ def _tier_choice_label(name: str, width: int, contents: str, cost: str) -> str: return f"{name}{' ' * (width - _cell_len(name))} · {contents} · {cost}" -def _step5_import(*, skip: bool, non_interactive: bool) -> object: +def _step6_import(*, skip: bool, non_interactive: bool) -> object: """Step 5 — optionally import conversation history from other AI tools.""" _step_header(6, _t("Import history from other AI tools", "从其他 AI 工具导入历史")) @@ -3417,7 +2853,7 @@ def _run_wizard_body( skip_test=skip_test, ), lambda: _step2_sandbox(skip=skip_sandbox, non_interactive=non_interactive), - lambda: _step3_channel(channel=channel, skip=skip_channel, non_interactive=non_interactive), + lambda: onboard_channels._step3_channel(channel=channel, skip=skip_channel, non_interactive=non_interactive), lambda: onboard_everos._step4_memory( skip=skip_memory, non_interactive=non_interactive, @@ -3430,7 +2866,7 @@ def _run_wizard_body( non_interactive=non_interactive, warnings=warnings, ), - lambda: _step5_import(skip=skip_import, non_interactive=non_interactive), + lambda: _step6_import(skip=skip_import, non_interactive=non_interactive), ] index = 0 diff --git a/tests/test_cli_onboard_commands.py b/tests/test_cli_onboard_commands.py index 0a51f3e1..6777480f 100644 --- a/tests/test_cli_onboard_commands.py +++ b/tests/test_cli_onboard_commands.py @@ -24,7 +24,7 @@ import typer from typer.testing import CliRunner -from raven.cli import onboard_commands, onboard_everos +from raven.cli import onboard_channels, onboard_commands, onboard_everos from raven.cli.commands import app from raven.config.loader import set_config_path @@ -651,10 +651,10 @@ def test_onboard_interactive_uses_stubbed_pickers( # Optional steps 2-4 are covered separately; no-op them here so the # interactive Step 1 path can be asserted without driving every screen. monkeypatch.setattr(onboard_commands, "_step2_sandbox", lambda **_: None) - monkeypatch.setattr(onboard_commands, "_step3_channel", lambda **_: None) + monkeypatch.setattr(onboard_channels, "_step3_channel", lambda **_: None) monkeypatch.setattr(onboard_everos, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) - monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) + monkeypatch.setattr(onboard_commands, "_step6_import", lambda **_: None) r = runner.invoke(app, ["onboard"]) assert r.exit_code == 0, r.stdout @@ -787,10 +787,10 @@ def _fake_autocomplete(message, choices, default=None, **kwargs): monkeypatch.setattr(questionary, "autocomplete", _fake_autocomplete) monkeypatch.setattr(onboard_commands, "_step2_sandbox", lambda **_: None) - monkeypatch.setattr(onboard_commands, "_step3_channel", lambda **_: None) + monkeypatch.setattr(onboard_channels, "_step3_channel", lambda **_: None) monkeypatch.setattr(onboard_everos, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) - monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) + monkeypatch.setattr(onboard_commands, "_step6_import", lambda **_: None) r = runner.invoke(app, ["onboard"]) assert r.exit_code == 0, r.stdout @@ -1647,7 +1647,7 @@ def test_custom_model_reuse_is_compatible( def test_channel_uses_interactive_login_real_specs() -> None: """Scancode channels (WhatsApp / WeChat) report interactive_login; others don't.""" - f = onboard_commands._channel_uses_interactive_login + f = onboard_channels._channel_uses_interactive_login assert f("whatsapp") is True assert f("weixin") is True assert f("telegram") is False @@ -1658,7 +1658,7 @@ def test_channel_order_overseas_common_before_domestic() -> None: (Reordered from the old domestic-first layout.) """ - names = onboard_commands._ordered_channel_names() + names = onboard_channels._ordered_channel_names() # US/global-common lead the list, ahead of the China-common group. for overseas in ("telegram", "discord", "slack", "whatsapp"): for domestic in ("weixin", "wecom", "feishu", "dingtalk", "qq"): @@ -1677,9 +1677,9 @@ def test_scancode_login_success_enables_channel(tmp_env: Path, monkeypatch: pyte _async_return(True), ) # Guard: the reflected-schema prompt must NOT be used for scancode channels. - monkeypatch.setattr(onboard_commands, "_prompt_channel_fields", _must_not_call("_prompt_channel_fields")) + monkeypatch.setattr(onboard_channels, "_prompt_channel_fields", _must_not_call("_prompt_channel_fields")) - onboard_commands._scancode_login("weixin") + onboard_channels._scancode_login("weixin") data = json.loads(tmp_env.read_text()) assert data["channels"]["weixin"]["enabled"] is True @@ -1698,7 +1698,7 @@ def test_scancode_login_retry_then_success(tmp_env: Path, monkeypatch: pytest.Mo "_failure_choice", lambda options, *, non_interactive: "retry", ) - onboard_commands._scancode_login("weixin") + onboard_channels._scancode_login("weixin") data = json.loads(tmp_env.read_text()) assert data["channels"]["weixin"]["enabled"] is True @@ -1715,7 +1715,7 @@ def test_scancode_login_skip_reverts_enable(tmp_env: Path, monkeypatch: pytest.M "_failure_choice", lambda options, *, non_interactive: "skip", ) - onboard_commands._scancode_login("weixin") + onboard_channels._scancode_login("weixin") data = json.loads(tmp_env.read_text()) # Not logged in → disabled, so it never falsely shows as connected. assert data["channels"]["weixin"]["enabled"] is False @@ -1724,18 +1724,18 @@ def test_scancode_login_skip_reverts_enable(tmp_env: Path, monkeypatch: pytest.M def test_add_one_channel_routes_scancode(tmp_env: Path, monkeypatch: pytest.MonkeyPatch) -> None: """`_add_one_channel` sends a scancode channel to login, NOT schema prompts.""" monkeypatch.setattr(onboard_commands, "_select_provider", lambda: "weixin") - monkeypatch.setattr(onboard_commands, "_select_channel", lambda: "weixin") + monkeypatch.setattr(onboard_channels, "_select_channel", lambda: "weixin") routed: list[str] = [] - monkeypatch.setattr(onboard_commands, "_scancode_login", lambda c, **kw: routed.append(c)) - monkeypatch.setattr(onboard_commands, "_prompt_channel_fields", _must_not_call("_prompt_channel_fields")) - onboard_commands._add_one_channel() + monkeypatch.setattr(onboard_channels, "_scancode_login", lambda c, **kw: routed.append(c)) + monkeypatch.setattr(onboard_channels, "_prompt_channel_fields", _must_not_call("_prompt_channel_fields")) + onboard_channels._add_one_channel() assert routed == ["weixin"] def test_scancode_login_node_missing_skip(tmp_env: Path, monkeypatch: pytest.MonkeyPatch) -> None: """WhatsApp with no Node/npm shows the install menu (NOT the QR menu); skip reverts the enable; the adapter's login is never called.""" - monkeypatch.setattr(onboard_commands, "_node_runtime_missing", lambda c: True) + monkeypatch.setattr(onboard_channels, "_node_runtime_missing", lambda c: True) # The Node-missing menu is distinct from the QR menu — assert its options # (no 're-show QR') and that login is never reached. captured: dict[str, list] = {} @@ -1749,7 +1749,7 @@ def _fc(options, *, non_interactive): "raven.channels.adapters.whatsapp.channel.WhatsAppChannel.login", _must_not_call("WhatsAppChannel.login"), ) - onboard_commands._scancode_login("whatsapp") + onboard_channels._scancode_login("whatsapp") data = json.loads(tmp_env.read_text()) # Not logged in → reverted to disabled. assert data["channels"]["whatsapp"]["enabled"] is False @@ -1761,7 +1761,7 @@ def _fc(options, *, non_interactive): def test_scancode_login_node_missing_retry_then_present(tmp_env: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Node-missing → 'retry' re-checks; once npm appears, login runs.""" missing = iter([True, False]) # first check missing, then present - monkeypatch.setattr(onboard_commands, "_node_runtime_missing", lambda c: next(missing)) + monkeypatch.setattr(onboard_channels, "_node_runtime_missing", lambda c: next(missing)) monkeypatch.setattr( onboard_commands, "_failure_choice", @@ -1771,7 +1771,7 @@ def test_scancode_login_node_missing_retry_then_present(tmp_env: Path, monkeypat "raven.channels.adapters.whatsapp.channel.WhatsAppChannel.login", _async_return(True), ) - onboard_commands._scancode_login("whatsapp") + onboard_channels._scancode_login("whatsapp") data = json.loads(tmp_env.read_text()) assert data["channels"]["whatsapp"]["enabled"] is True @@ -1853,10 +1853,10 @@ def _s3(**_): monkeypatch.setattr(onboard_commands, "_bootstrap_empty_config", lambda: None) monkeypatch.setattr(onboard_commands, "_step1_provider", _s1) monkeypatch.setattr(onboard_commands, "_step2_sandbox", _s2) - monkeypatch.setattr(onboard_commands, "_step3_channel", _s3) + monkeypatch.setattr(onboard_channels, "_step3_channel", _s3) monkeypatch.setattr(onboard_everos, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) - monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) + monkeypatch.setattr(onboard_commands, "_step6_import", lambda **_: None) onboard_commands.run_wizard(non_interactive=False) # s2 returns BACK once → s1 replays → s2 again → forward. @@ -1882,10 +1882,10 @@ def test_first_screen_back_does_not_skip_step1( monkeypatch.setattr(onboard_commands, "_pick_model", lambda provider, spec, **_: spec.default_model) # Optional steps are no-ops here; we only assert Step 1 wasn't skipped. monkeypatch.setattr(onboard_commands, "_step2_sandbox", lambda **_: None) - monkeypatch.setattr(onboard_commands, "_step3_channel", lambda **_: None) + monkeypatch.setattr(onboard_channels, "_step3_channel", lambda **_: None) monkeypatch.setattr(onboard_everos, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) - monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) + monkeypatch.setattr(onboard_commands, "_step6_import", lambda **_: None) onboard_commands.run_wizard(non_interactive=False) @@ -1929,10 +1929,10 @@ def _verify(name, *a, **kw): # On the failure submenu, choose "switch". monkeypatch.setattr(onboard_commands, "_failure_choice", lambda options, *, non_interactive: "switch") monkeypatch.setattr(onboard_commands, "_step2_sandbox", lambda **_: None) - monkeypatch.setattr(onboard_commands, "_step3_channel", lambda **_: None) + monkeypatch.setattr(onboard_channels, "_step3_channel", lambda **_: None) monkeypatch.setattr(onboard_everos, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) - monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) + monkeypatch.setattr(onboard_commands, "_step6_import", lambda **_: None) # Should complete (not raise typer.Exit) — steps 2/3/4 ran. onboard_commands.run_wizard(non_interactive=False) @@ -1964,10 +1964,10 @@ def _fake_prompt_api_key(provider, **kw): monkeypatch.setattr(onboard_commands, "_prompt_api_key", _fake_prompt_api_key) monkeypatch.setattr(onboard_commands, "_pick_model", lambda provider, spec, **_: spec.default_model) monkeypatch.setattr(onboard_commands, "_step2_sandbox", lambda **_: None) - monkeypatch.setattr(onboard_commands, "_step3_channel", lambda **_: None) + monkeypatch.setattr(onboard_channels, "_step3_channel", lambda **_: None) monkeypatch.setattr(onboard_everos, "_step4_memory", lambda **_: None) monkeypatch.setattr(onboard_commands, "_step5_deep_research", lambda **_: None) - monkeypatch.setattr(onboard_commands, "_step5_import", lambda **_: None) + monkeypatch.setattr(onboard_commands, "_step6_import", lambda **_: None) onboard_commands.run_wizard(non_interactive=False) @@ -2256,7 +2256,7 @@ def ask(self) -> str: monkeypatch.setattr(questionary, "text", lambda label, **kw: _Prompt(label, **kw)) monkeypatch.setattr(questionary, "password", lambda label, **kw: _Prompt(label, **kw)) - onboard_commands._prompt_channel_fields("feishu") + onboard_channels._prompt_channel_fields("feishu") # promptable order: app_id, app_secret (both required), encrypt_key, verification_token (optional) def _ph_text(placeholder: Any) -> Any: @@ -2454,7 +2454,7 @@ def _run_import_step( "raven.cli.import_commands._build_and_run", AsyncMock(return_value=_no_op_import_result()), ) - onboard_commands._step5_import(skip=False, non_interactive=False) + onboard_commands._step6_import(skip=False, non_interactive=False) return scripted @@ -2497,7 +2497,7 @@ def test_import_step_installs_skills_when_the_scan_finds_nothing( monkeypatch.setattr("raven.importer.scanners.scan_all", AsyncMock(return_value=[])) _patch_skills_only_install(monkeypatch, tmp_path) - onboard_commands._step5_import(skip=False, non_interactive=False) + onboard_commands._step6_import(skip=False, non_interactive=False) out = " ".join(capsys.readouterr().out.split()) assert "12 installed" in out, out @@ -2535,7 +2535,7 @@ def test_import_step_installs_skills_when_the_tier_keeps_nothing( _patch_skill_count(monkeypatch, 12) _patch_skills_only_install(monkeypatch, tmp_path) - onboard_commands._step5_import(skip=False, non_interactive=False) + onboard_commands._step6_import(skip=False, non_interactive=False) out = " ".join(capsys.readouterr().out.split()) assert "12 installed" in out, out @@ -2556,7 +2556,7 @@ def test_the_wizard_asks_before_copying_a_skill_tree( monkeypatch.setattr("raven.importer.scanners.scan_all", AsyncMock(return_value=[])) installer = _patch_skills_only_install(monkeypatch, tmp_path, confirm=False) - onboard_commands._step5_import(skip=False, non_interactive=False) + onboard_commands._step6_import(skip=False, non_interactive=False) installer.assert_not_awaited() out = " ".join(capsys.readouterr().out.split()) From cdc2ec6981d45852e7f8bc987f848db2c140059a Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 00:45:44 +0800 Subject: [PATCH 25/78] feat(*): route multi-endpoint sections through the rotor Third stage of multi-endpoint support. make_provider builds an EndpointRotorProvider when a section resolves to more than one endpoint, leaves the single-endpoint path byte-identical, and refuses the endpoints field on codex/minimax_oauth/azure and OAuth sections at startup -- those connect through one signed-in account, so accepting the field and using only the first entry would be the declared-but-dead shape this feature exists to kill. Credential gates now look one level down: a key on any endpoint satisfies the same requirement a flat key does, through the one _present helper that routing, display and startup all share. ProviderConfig gains endpoint_strategy (sticky|round_robin). Co-authored-by: Claude (claude-fable-5) --- raven/cli/_helpers.py | 59 +++++++++++++++---- raven/config/schema.py | 4 ++ raven/providers/auth.py | 16 ++++- tests/test_cli_helpers.py | 93 ++++++++++++++++++++++++++++++ tests/test_config_schema.py | 17 ++++++ tests/test_provider_auth_method.py | 47 +++++++++++++++ 6 files changed, 224 insertions(+), 12 deletions(-) diff --git a/raven/cli/_helpers.py b/raven/cli/_helpers.py index f6571d26..8d9b4e3a 100644 --- a/raven/cli/_helpers.py +++ b/raven/cli/_helpers.py @@ -103,6 +103,7 @@ def check_provider_credentials(config: Config) -> None: def make_provider(config: Config): """Create the appropriate LLM provider from config.""" + from raven.providers.auth import MissingCredentialsError from raven.providers.azure_openai_provider import AzureOpenAIProvider from raven.providers.base import GenerationSettings from raven.providers.openai_codex_provider import OpenAICodexProvider @@ -118,6 +119,21 @@ def make_provider(config: Config): spec = find_by_name(provider_name) if provider_name else None client = spec.client if spec else "" + # codex / minimax_oauth / azure each need more than a key and an address + # (a device-flow token, a deployment path, ...), and an OAuth section + # (spec.is_oauth, e.g. github_copilot -- which has no dedicated client and + # falls to the litellm branch below) connects through one signed-in + # account, not several. `endpoints` is meaningful only for a plain + # API-key vendor reached through litellm, so a section combining it with + # any of these is rejected here rather than silently using just the first + # entry. + if p and p.endpoints and (client in {"codex", "minimax_oauth", "azure"} or (spec is not None and spec.is_oauth)): + raise MissingCredentialsError( + f"{provider_name} does not support multiple endpoints -- remove the `endpoints` " + "field from its config; this provider connects through a single account, not several", + provider=provider_name or "", + ) + if client == "codex": provider = OpenAICodexProvider(default_model=model) elif client == "minimax_oauth": @@ -137,18 +153,41 @@ def make_provider(config: Config): ) else: from raven.providers.capabilities import wire_overrides + from raven.providers.endpoints import provider_endpoints from raven.providers.litellm_provider import LiteLLMProvider - extra_body = wire_overrides(provider_name, model) or None - provider = LiteLLMProvider( - api_key=p.effective_api_key if p else None, - api_base=config.get_api_base(model), - default_model=model, - extra_headers=p.extra_headers if p else None, - provider_name=provider_name, - extra_body=extra_body, - model_overrides=config.agents.defaults.model_overrides, - ) + eps = provider_endpoints(p) if p else [] + if len(eps) > 1: + from raven.providers.endpoint_rotor import EndpointRotorProvider + + def make_inner(ep): + return LiteLLMProvider( + api_key=ep.api_key, + api_base=ep.api_base or config.get_api_base(model), + default_model=model, + extra_headers=ep.extra_headers or (p.extra_headers if p else None), + provider_name=provider_name, + extra_body=wire_overrides(provider_name, model) or None, + model_overrides=config.agents.defaults.model_overrides, + ) + + provider = EndpointRotorProvider( + eps, + make_inner, + default_model=model, + strategy=p.endpoint_strategy if p else "sticky", + ) + else: + extra_body = wire_overrides(provider_name, model) or None + provider = LiteLLMProvider( + api_key=p.effective_api_key if p else None, + api_base=config.get_api_base(model), + default_model=model, + extra_headers=p.extra_headers if p else None, + provider_name=provider_name, + extra_body=extra_body, + model_overrides=config.agents.defaults.model_overrides, + ) defaults = config.agents.defaults provider.generation = GenerationSettings( diff --git a/raven/config/schema.py b/raven/config/schema.py index e1c0f208..65298242 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -372,6 +372,10 @@ class ProviderConfig(Base): # one place that resolves which of the two shapes (or Gemini's # `api_key_list`) is in effect. endpoints: list[ProviderEndpoint] = Field(default_factory=list) + # How requests spread across `endpoints` when there is more than one: + # "sticky" keeps using the first healthy entry until it fails, "round_robin" + # cycles through all of them. Meaningless with zero or one endpoint. + endpoint_strategy: Literal["sticky", "round_robin"] = "sticky" # Keyed by model id, in any spelling: what the user knows about a model that # the catalogues do not. Deliberately additive rather than a change to # `models` -- that list already lets a model be added, and what was missing diff --git a/raven/providers/auth.py b/raven/providers/auth.py index 77462ba2..3f820a2d 100644 --- a/raven/providers/auth.py +++ b/raven/providers/auth.py @@ -130,13 +130,25 @@ def _present(section: Any, name: str) -> bool: Sections reach here as both: the schema object on the routing path, a raw mapping on the display path. + + A section holding ``endpoints`` instead of the flat fields is checked the + same way, one level down: if the flat field is unset, any endpoint that has + ``name`` set also counts. A section with several endpoints and a key on + only one of them is exactly as usable as one with a single flat key -- + routing and startup both read the resolved list (``provider_endpoints``), + not the flat field, so a gate that only looked at the flat field would + reject a section its own request path can serve. """ if section is None: return False value = section.get(name) if isinstance(section, dict) else getattr(section, name, None) if isinstance(value, (list, tuple)): - return any(bool(v) for v in value) - return bool(value) + if any(bool(v) for v in value): + return True + elif value: + return True + endpoints = section.get("endpoints") if isinstance(section, dict) else getattr(section, "endpoints", None) + return any(_present(endpoint, name) for endpoint in endpoints or []) def _token_present(provider: str) -> bool: diff --git a/tests/test_cli_helpers.py b/tests/test_cli_helpers.py index 728e125e..d2a5ac25 100644 --- a/tests/test_cli_helpers.py +++ b/tests/test_cli_helpers.py @@ -252,3 +252,96 @@ def test_the_credential_check_wants_both_halves_of_an_azure_endpoint(tmp_path: P with pytest.raises(MissingCredentialsError): _helpers.check_provider_credentials(load_config(cfg)) + + +# --------------------------------------------------------------------------- +# make_provider — several endpoints under one section (S3) +# --------------------------------------------------------------------------- + + +def test_make_provider_builds_a_rotor_over_several_endpoints(monkeypatch: pytest.MonkeyPatch) -> None: + """More than one endpoint fans out behind an ``EndpointRotorProvider``, + one inner ``LiteLLMProvider`` per entry.""" + from raven.config.schema import Config + from raven.providers.endpoint_rotor import EndpointRotorProvider + from raven.providers.litellm_provider import LiteLLMProvider + + config = Config.model_validate( + { + "providers": { + "custom": { + "endpoints": [ + {"label": "a", "apiKey": "k1", "apiBase": "https://a.example"}, + {"label": "b", "apiKey": "k2", "apiBase": "https://b.example"}, + ] + } + }, + "agents": {"defaults": {"model": "my-model", "provider": "custom"}}, + } + ) + monkeypatch.setattr("raven.cli._helpers.check_provider_credentials", lambda _config: None) + + provider = _helpers.make_provider(config) + + assert isinstance(provider, EndpointRotorProvider) + assert len(provider._inners) == 2 + assert all(isinstance(inner, LiteLLMProvider) for inner in provider._inners) + + +def test_make_provider_a_single_endpoint_entry_still_returns_a_plain_provider( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """One entry in ``endpoints`` takes the unchanged single-provider path, + same as the flat ``apiKey``/``apiBase`` fields -- no rotor for one.""" + from raven.config.schema import Config + from raven.providers.litellm_provider import LiteLLMProvider + + config = Config.model_validate( + { + "providers": {"custom": {"endpoints": [{"label": "only", "apiKey": "k1"}]}}, + "agents": {"defaults": {"model": "my-model", "provider": "custom"}}, + } + ) + monkeypatch.setattr("raven.cli._helpers.check_provider_credentials", lambda _config: None) + + provider = _helpers.make_provider(config) + + assert type(provider) is LiteLLMProvider + + +@pytest.mark.parametrize( + ("provider", "model", "extra_section"), + [ + ("openai_codex", "openai-codex/gpt-5.3-codex", {}), + ("minimax_global", "minimax-global/MiniMax-M3", {}), + ("azure_openai", "azure_openai/my-deployment", {}), + ("github_copilot", "github_copilot/gpt-4o", {}), + ], +) +def test_make_provider_rejects_endpoints_on_providers_that_cannot_rotate( + provider: str, + model: str, + extra_section: dict, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """codex / minimax_oauth / azure need more than a key and an address, and + an OAuth section connects through one signed-in account -- ``endpoints`` + on any of them is a configuration error at construction time, not a + silently-ignored field.""" + from raven.config.schema import Config + from raven.providers.auth import MissingCredentialsError + + section = { + "endpoints": [{"label": "a", "apiKey": "k1"}, {"label": "b", "apiKey": "k2"}], + **extra_section, + } + config = Config.model_validate( + { + "providers": {provider: section}, + "agents": {"defaults": {"model": model, "provider": provider}}, + } + ) + monkeypatch.setattr("raven.cli._helpers.check_provider_credentials", lambda _config: None) + + with pytest.raises(MissingCredentialsError, match="endpoints"): + _helpers.make_provider(config) diff --git a/tests/test_config_schema.py b/tests/test_config_schema.py index 7b04d666..2df6db14 100644 --- a/tests/test_config_schema.py +++ b/tests/test_config_schema.py @@ -14,6 +14,9 @@ from __future__ import annotations +import pytest +from pydantic import ValidationError + from raven.config.schema import AgentDefaults, ProviderConfig, ProviderEndpoint @@ -67,3 +70,17 @@ def test_provider_config_endpoints_round_trip_with_camel_alias() -> None: dumped = section.model_dump(by_alias=True) assert dumped["endpoints"][0]["apiKey"] == "sk-1" assert dumped["endpoints"][1]["label"] == "backup" + + +def test_endpoint_strategy_defaults_to_sticky() -> None: + assert ProviderConfig().endpoint_strategy == "sticky" + + +def test_endpoint_strategy_accepts_round_robin_with_camel_alias() -> None: + section = ProviderConfig.model_validate({"endpointStrategy": "round_robin"}) + assert section.endpoint_strategy == "round_robin" + + +def test_endpoint_strategy_rejects_an_unknown_value() -> None: + with pytest.raises(ValidationError): + ProviderConfig.model_validate({"endpointStrategy": "random"}) diff --git a/tests/test_provider_auth_method.py b/tests/test_provider_auth_method.py index 32f8ed5a..d8940f04 100644 --- a/tests/test_provider_auth_method.py +++ b/tests/test_provider_auth_method.py @@ -83,6 +83,16 @@ "model": "ollama_chat/llama3.2", "section": {}, }, + "anthropic_endpoints_only": { + "provider": "anthropic", + "model": "anthropic/claude-sonnet-5", + "section": {"endpoints": [{"label": "primary", "apiKey": "sk-ant-TEST"}]}, + }, + "anthropic_endpoints_all_keys_empty": { + "provider": "anthropic", + "model": "anthropic/claude-sonnet-5", + "section": {"endpoints": [{"label": "primary", "apiKey": ""}, {"label": "backup", "apiKey": ""}]}, + }, } @@ -194,6 +204,43 @@ def test_a_key_in_the_plural_field_is_a_configured_provider(tmp_path: Path) -> N assert _startup_says(case, path) +def test_a_key_in_an_endpoints_entry_is_a_configured_provider(tmp_path: Path) -> None: + """An endpoints-only section is exactly as usable as a flat key -- routing and + startup both read the resolved list (``provider_endpoints``), not the flat + field, so a gate that only looked at the flat field would reject a section + its own request path can serve. + """ + case = SCENARIOS["anthropic_endpoints_only"] + path = _config_file(tmp_path, case) + assert _display_says(case, path) + assert _routing_says(case, path) + assert _startup_says(case, path) + + +def test_an_endpoints_list_with_every_key_empty_is_not_configured(tmp_path: Path) -> None: + case = SCENARIOS["anthropic_endpoints_all_keys_empty"] + path = _config_file(tmp_path, case) + assert not _display_says(case, path) + assert not _routing_says(case, path) + assert not _startup_says(case, path) + + +def test_credential_status_ok_for_an_endpoints_only_section() -> None: + from raven.config.schema import ProviderConfig + from raven.providers.auth import credential_status + + section = ProviderConfig.model_validate({"endpoints": [{"label": "a", "apiKey": "sk-1"}]}) + assert credential_status("anthropic", section).ok + + +def test_credential_status_not_ok_when_every_endpoint_key_is_empty() -> None: + from raven.config.schema import ProviderConfig + from raven.providers.auth import credential_status + + section = ProviderConfig.model_validate({"endpoints": [{"label": "a", "apiKey": ""}, {"label": "b", "apiKey": ""}]}) + assert not credential_status("anthropic", section).ok + + def test_a_provider_whose_key_lives_in_a_list_sends_a_key(tmp_path: Path) -> None: """Passing the gate is not enough; the request has to carry a credential. From 5c94ea1c757963685df248c3dac42b1d46750b44 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 00:47:30 +0800 Subject: [PATCH 26/78] test(providers): argue the two new key-reading files into the guard The AST guard that keeps configuredness decisions inside providers.auth flags every credential-field read outside an argued allowlist. Two recent changes read keys legitimately and slipped past every executor's test selection: onboard_everos.py carries the wizard cluster split out of the already-allowed onboard_commands, and providers/endpoints.py is the connection-material reading layer itself. Both entries now carry their argument in the list. Co-authored-by: Claude (claude-fable-5) --- tests/test_provider_auth_method.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_provider_auth_method.py b/tests/test_provider_auth_method.py index d8940f04..fec7604f 100644 --- a/tests/test_provider_auth_method.py +++ b/tests/test_provider_auth_method.py @@ -296,6 +296,14 @@ def test_only_the_auth_module_decides_configuredness_from_a_key() -> None: "raven/providers/litellm_provider.py", "raven/cli/_helpers.py", "raven/cli/onboard_commands.py", + # Carries the wizard's EverOS cluster split out of onboard_commands -- + # same reads, same argument, new file name. + "raven/cli/onboard_everos.py", + # The connection-material reading layer itself: resolves flat fields, + # api_key_list and endpoints into one list for whoever sends requests. + # Configuredness still rules through auth, which consults this shape + # via its own _present. + "raven/providers/endpoints.py", "raven/cli/provider_commands.py", "raven/cli/status_commands.py", "raven/tui_rpc/methods/model.py", From b8baa25c0e5493594d6b121b448afb0d0e983fd3 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 00:49:29 +0800 Subject: [PATCH 27/78] feat(*): manage provider endpoints from the cli Fourth stage of multi-endpoint support: add/remove/list ops in update_providers following the provider-model pair's read-validate- mutate-write pattern -- label is the idempotency key, so re-running add with a rotated api_key writes the rotation; remove of an absent label is the same silent no-op remove_provider_model settled on; list redacts the key like every other secret field. raven provider endpoint add|remove|list exposes them with the file's flag conventions. Co-authored-by: Claude (claude-fable-5) --- raven/cli/provider_commands.py | 115 +++++++++++++++++++++++ raven/config/update_providers.py | 98 +++++++++++++++++++- tests/test_cli_provider_commands.py | 126 ++++++++++++++++++++++++++ tests/test_config_update_providers.py | 118 ++++++++++++++++++++++++ 4 files changed, 456 insertions(+), 1 deletion(-) diff --git a/raven/cli/provider_commands.py b/raven/cli/provider_commands.py index 312a5755..5db79a34 100644 --- a/raven/cli/provider_commands.py +++ b/raven/cli/provider_commands.py @@ -15,6 +15,16 @@ also lose their token file - ``provider show `` — reflect available ``--flag`` fields +Endpoint subcommands (``provider endpoint ...``) manage a plain API-key +provider's ``endpoints`` list -- several full key/base/header groups under one +section, for a vendor reachable by more than one account or region. OAuth +providers and Azure OpenAI / OpenAI Codex reject this at startup; only vendors +reached through the plain LiteLLM client accept it: + +- ``provider endpoint add --label X --api-key ... [--api-base ...]`` +- ``provider endpoint remove --label X`` +- ``provider endpoint list `` + Architecture: write operations go ONLY through :mod:`raven.config.update_providers`. Command bodies do not import ``load_config`` / ``save_config`` / provider Pydantic classes. @@ -25,6 +35,7 @@ from __future__ import annotations +import json import os import sys from typing import Any @@ -691,4 +702,108 @@ def provider_show_cmd( _register_config_commands(provider_app) +endpoint_app = typer.Typer( + help=( + "Manage a provider's endpoints -- several full key/base/header groups " + "under one section, for a vendor reachable by more than one account or " + "region. Only plain API-key providers accept this: a provider using " + "OAuth, or Azure OpenAI / OpenAI Codex, is rejected at startup if it has " + "any configured." + ) +) + + +def _parse_extra_headers(value: str) -> dict[str, str] | None: + """Parse ``--extra-headers`` JSON into a dict, or None when unset.""" + if not value: + return None + try: + parsed = json.loads(value) + except json.JSONDecodeError: + raise typer.BadParameter('--extra-headers must be a JSON object, e.g. \'{"X-Foo": "bar"}\'') + if not isinstance(parsed, dict): + raise typer.BadParameter("--extra-headers must be a JSON object") + return parsed + + +@endpoint_app.command("add") +def endpoint_add_cmd( + name: str = typer.Argument(..., help="Provider name (e.g. openrouter)"), + label: str = typer.Option(..., "--label", help="Idempotency key: an existing label is replaced, not merged"), + api_key: str = typer.Option(..., "--api-key", help="API key for this endpoint"), + api_base: str = typer.Option("", "--api-base", help="Base URL for this endpoint"), + extra_headers: str = typer.Option("", "--extra-headers", help='Extra headers as JSON, e.g. {"X-Foo": "bar"}'), +): + """Add or replace one endpoint on a provider, keyed by ``--label``. + + Only meaningful for plain API-key providers reached through the LiteLLM + client -- a provider using OAuth, or Azure OpenAI / OpenAI Codex, refuses + to start with any endpoints configured. + """ + from raven.config.update_providers import add_provider_endpoint + + headers = _parse_extra_headers(extra_headers) + try: + endpoints = add_provider_endpoint( + name, + label=label, + api_key=api_key, + api_base=api_base or None, + extra_headers=headers, + ) + except KeyError as exc: + console.print(f"[red]✗[/red] {exc}") + raise typer.Exit(1) + + console.print(f"[green]✓[/green] {name} endpoint {label!r} saved ({len(endpoints)} total)") + + +@endpoint_app.command("remove") +def endpoint_remove_cmd( + name: str = typer.Argument(..., help="Provider name"), + label: str = typer.Option(..., "--label", help="Label of the endpoint to remove"), +): + """Remove one endpoint by ``--label`` (no-op if the label is not present).""" + from raven.config.update_providers import remove_provider_endpoint + + try: + endpoints = remove_provider_endpoint(name, label) + except KeyError as exc: + console.print(f"[red]✗[/red] {exc}") + raise typer.Exit(1) + + console.print(f"[green]✓[/green] {name} endpoint {label!r} removed ({len(endpoints)} remaining)") + + +@endpoint_app.command("list") +def endpoint_list_cmd( + name: str = typer.Argument(..., help="Provider name"), +): + """List a provider's endpoints. API keys redacted.""" + from raven.config.update_providers import list_provider_endpoints + + try: + endpoints = list_provider_endpoints(name) + except KeyError as exc: + console.print(f"[red]✗[/red] {exc}") + raise typer.Exit(1) + + table = Table(title=f"Provider endpoints: {name}") + table.add_column("Label", style="cyan", no_wrap=True) + table.add_column("API Key") + table.add_column("API Base", overflow="fold") + table.add_column("Extra Headers", overflow="fold") + for ep in endpoints: + table.add_row( + ep["label"], + ep["api_key"], + ep["api_base"] or "", + str(ep["extra_headers"]) if ep["extra_headers"] else "", + ) + console.print(table) + + +provider_app.add_typer(endpoint_app, name="endpoint") + + __all__ = ["provider_app"] diff --git a/raven/config/update_providers.py b/raven/config/update_providers.py index 5842e3ee..e8c34c44 100644 --- a/raven/config/update_providers.py +++ b/raven/config/update_providers.py @@ -29,7 +29,7 @@ from pydantic_core import PydanticUndefined from raven.config.loader import get_config_path, read_raw_or_raise -from raven.config.schema import ProviderConfig, ProvidersConfig +from raven.config.schema import ProviderConfig, ProviderEndpoint, ProvidersConfig from raven.providers.registry import ( ProviderSpec, canonical_provider_name, @@ -845,6 +845,99 @@ def remove_provider_model( return models +def _load_provider_endpoints(name: str, data: dict[str, Any]) -> tuple[type, list[ProviderEndpoint]]: + cls = _provider_schema_cls(name) + section = _raw_section(data, name) + try: + instance = cls.model_validate(section) + except ValidationError: + instance = cls() + return cls, list(getattr(instance, "endpoints", []) or []) + + +def add_provider_endpoint( + name: str, + *, + label: str, + api_key: str = "", + api_base: str | None = None, + extra_headers: dict[str, str] | None = None, + config_path: Path | None = None, +) -> list[ProviderEndpoint]: + """Add or replace one entry in a provider's ``endpoints`` list, keyed by ``label``. + + ``label`` is the idempotency key ``ProviderEndpoint`` declares it as: an + existing entry with that label is replaced wholesale, not merged field by + field, so re-running this with a rotated ``api_key`` is how the rotation + gets written. A new label appends. + + Returns the new endpoint list. Raises KeyError for an unknown provider. + """ + name = canonical_provider_name(name) + path = config_path or get_config_path() + data = read_raw_or_raise(path) + cls, endpoints = _load_provider_endpoints(name, data) + + new_endpoint = ProviderEndpoint(label=label, api_key=api_key, api_base=api_base, extra_headers=extra_headers) + updated = [new_endpoint if ep.label == label else ep for ep in endpoints] + if not any(ep.label == label for ep in endpoints): + updated.append(new_endpoint) + + section = _raw_section(data, name) + section["endpoints"] = [ep.model_dump(by_alias=True) for ep in updated] + validated = cls.model_validate(section) + _write_raw_section(data, name, validated.model_dump(by_alias=True)) + _write_atomic(path, data) + return updated + + +def remove_provider_endpoint( + name: str, + label: str, + *, + config_path: Path | None = None, +) -> list[ProviderEndpoint]: + """Remove one endpoint by ``label`` (no-op if absent, mirrors ``remove_provider_model``). + + Returns the new endpoint list. Raises KeyError for an unknown provider. + """ + name = canonical_provider_name(name) + path = config_path or get_config_path() + data = read_raw_or_raise(path) + cls, endpoints = _load_provider_endpoints(name, data) + + remaining = [ep for ep in endpoints if ep.label != label] + if len(remaining) != len(endpoints): + section = _raw_section(data, name) + section["endpoints"] = [ep.model_dump(by_alias=True) for ep in remaining] + validated = cls.model_validate(section) + _write_raw_section(data, name, validated.model_dump(by_alias=True)) + _write_atomic(path, data) + return remaining + + +def list_provider_endpoints(name: str, *, config_path: Path | None = None) -> list[dict[str, Any]]: + """List a provider's ``endpoints``, ``api_key`` redacted for display. + + Returns one dict per endpoint: ``label``, ``api_key`` (``****set****`` / + ``(empty)``, same rule as every other secret field), ``api_base``, + ``extra_headers``. Raises KeyError for an unknown provider. + """ + name = canonical_provider_name(name) + path = config_path or get_config_path() + data = read_raw_or_raise(path) + _, endpoints = _load_provider_endpoints(name, data) + return [ + { + "label": ep.label, + "api_key": _redact(ep.api_key), + "api_base": ep.api_base, + "extra_headers": ep.extra_headers, + } + for ep in endpoints + ] + + # --------------------------------------------------------------------------- # Public API: credential health check # --------------------------------------------------------------------------- @@ -1316,5 +1409,8 @@ def _probe_codex_catalog(*, timeout_s: float) -> dict[str, Any]: "get_provider_config", "set_provider_fields", "reset_provider", + "add_provider_endpoint", + "remove_provider_endpoint", + "list_provider_endpoints", "test_provider", ] diff --git a/tests/test_cli_provider_commands.py b/tests/test_cli_provider_commands.py index beb44d86..0ee6949e 100644 --- a/tests/test_cli_provider_commands.py +++ b/tests/test_cli_provider_commands.py @@ -853,3 +853,129 @@ def test_use_still_pins_a_vendor_that_is_configured(tmp_config: Path) -> None: r = runner.invoke(app, ["provider", "use", "anthropic/claude-sonnet-4-5"]) assert r.exit_code == 0, r.output assert json.loads(tmp_config.read_text(encoding="utf-8"))["agents"]["defaults"]["provider"] == "anthropic" + + +# --------------------------------------------------------------------------- +# provider endpoint add / remove / list +# --------------------------------------------------------------------------- + + +def test_endpoint_help_lists_subcommands() -> None: + r = runner.invoke(app, ["provider", "endpoint", "--help"]) + assert r.exit_code == 0 + assert "add" in r.stdout + assert "remove" in r.stdout + assert "list" in r.stdout + + +def test_endpoint_add_writes_the_section(tmp_config: Path) -> None: + r = runner.invoke( + app, + ["provider", "endpoint", "add", "openrouter", "--label", "primary", "--api-key", "k1"], + ) + assert r.exit_code == 0, r.output + assert "primary" in r.stdout + + section = json.loads(tmp_config.read_text(encoding="utf-8"))["providers"]["openrouter"] + assert section["endpoints"] == [{"label": "primary", "apiKey": "k1", "apiBase": None, "extraHeaders": None}] + + +def test_endpoint_add_with_api_base_and_headers(tmp_config: Path) -> None: + r = runner.invoke( + app, + [ + "provider", + "endpoint", + "add", + "openrouter", + "--label", + "eu", + "--api-key", + "k1", + "--api-base", + "https://eu.example.com", + "--extra-headers", + '{"X-Region": "eu"}', + ], + ) + assert r.exit_code == 0, r.output + + section = json.loads(tmp_config.read_text(encoding="utf-8"))["providers"]["openrouter"] + assert section["endpoints"][0]["apiBase"] == "https://eu.example.com" + assert section["endpoints"][0]["extraHeaders"] == {"X-Region": "eu"} + + +def test_endpoint_add_rejects_malformed_headers_json(tmp_config: Path) -> None: + r = runner.invoke( + app, + ["provider", "endpoint", "add", "openrouter", "--label", "x", "--api-key", "k", "--extra-headers", "{not-json"], + ) + assert r.exit_code != 0 + assert "JSON" in r.output + + +def test_endpoint_add_same_label_replaces(tmp_config: Path) -> None: + runner.invoke(app, ["provider", "endpoint", "add", "openrouter", "--label", "primary", "--api-key", "k1"]) + r = runner.invoke(app, ["provider", "endpoint", "add", "openrouter", "--label", "primary", "--api-key", "k2"]) + assert r.exit_code == 0, r.output + + section = json.loads(tmp_config.read_text(encoding="utf-8"))["providers"]["openrouter"] + assert len(section["endpoints"]) == 1 + assert section["endpoints"][0]["apiKey"] == "k2" + + +def test_endpoint_add_unknown_provider_exits_1(tmp_config: Path) -> None: + r = runner.invoke( + app, + ["provider", "endpoint", "add", "no-such-provider", "--label", "x", "--api-key", "k"], + ) + assert r.exit_code == 1 + assert "Unknown provider" in r.output + + +def test_endpoint_remove_drops_the_label(tmp_config: Path) -> None: + runner.invoke(app, ["provider", "endpoint", "add", "openrouter", "--label", "primary", "--api-key", "k1"]) + runner.invoke(app, ["provider", "endpoint", "add", "openrouter", "--label", "backup", "--api-key", "k2"]) + + r = runner.invoke(app, ["provider", "endpoint", "remove", "openrouter", "--label", "primary"]) + assert r.exit_code == 0, r.output + + section = json.loads(tmp_config.read_text(encoding="utf-8"))["providers"]["openrouter"] + assert [e["label"] for e in section["endpoints"]] == ["backup"] + + +def test_endpoint_remove_absent_label_is_noop(tmp_config: Path) -> None: + runner.invoke(app, ["provider", "endpoint", "add", "openrouter", "--label", "primary", "--api-key", "k1"]) + + r = runner.invoke(app, ["provider", "endpoint", "remove", "openrouter", "--label", "not-there"]) + assert r.exit_code == 0, r.output + + section = json.loads(tmp_config.read_text(encoding="utf-8"))["providers"]["openrouter"] + assert [e["label"] for e in section["endpoints"]] == ["primary"] + + +def test_endpoint_remove_unknown_provider_exits_1(tmp_config: Path) -> None: + r = runner.invoke(app, ["provider", "endpoint", "remove", "no-such-provider", "--label", "x"]) + assert r.exit_code == 1 + assert "Unknown provider" in r.output + + +def test_endpoint_list_redacts_api_key(tmp_config: Path) -> None: + runner.invoke(app, ["provider", "endpoint", "add", "openrouter", "--label", "primary", "--api-key", "k1"]) + + r = runner.invoke(app, ["provider", "endpoint", "list", "openrouter"]) + assert r.exit_code == 0, r.output + assert "primary" in r.stdout + assert "****set****" in r.stdout + assert "k1" not in r.stdout + + +def test_endpoint_list_empty_when_none_configured(tmp_config: Path) -> None: + r = runner.invoke(app, ["provider", "endpoint", "list", "openrouter"]) + assert r.exit_code == 0, r.output + + +def test_endpoint_list_unknown_provider_exits_1(tmp_config: Path) -> None: + r = runner.invoke(app, ["provider", "endpoint", "list", "no-such-provider"]) + assert r.exit_code == 1 + assert "Unknown provider" in r.output diff --git a/tests/test_config_update_providers.py b/tests/test_config_update_providers.py index 239d64ee..7ca08cdb 100644 --- a/tests/test_config_update_providers.py +++ b/tests/test_config_update_providers.py @@ -13,10 +13,13 @@ from raven.config.update_providers import ( _copilot_token_dir, _oauth_token_path, + add_provider_endpoint, add_provider_model, get_provider_config, + list_provider_endpoints, list_providers, provider_field_specs, + remove_provider_endpoint, remove_provider_model, reset_provider, set_provider_fields, @@ -621,6 +624,121 @@ def test_add_provider_model_unknown_provider_raises(cfg_path: Path) -> None: add_provider_model("nonexistent_provider", "x", config_path=cfg_path) +# --------------------------------------------------------------------------- +# Endpoints (add_provider_endpoint / remove_provider_endpoint / list_provider_endpoints) +# --------------------------------------------------------------------------- + + +def test_add_provider_endpoint_appends(cfg_path: Path) -> None: + endpoints = add_provider_endpoint("openrouter", label="primary", api_key="k1", config_path=cfg_path) + + assert [e.label for e in endpoints] == ["primary"] + section = _read(cfg_path)["providers"]["openrouter"] + assert section["endpoints"] == [{"label": "primary", "apiKey": "k1", "apiBase": None, "extraHeaders": None}] + + +def test_add_provider_endpoint_appends_a_second_label(cfg_path: Path) -> None: + add_provider_endpoint("openrouter", label="primary", api_key="k1", config_path=cfg_path) + endpoints = add_provider_endpoint("openrouter", label="backup", api_key="k2", config_path=cfg_path) + + assert [e.label for e in endpoints] == ["primary", "backup"] + + +def test_add_provider_endpoint_same_label_replaces_wholesale(cfg_path: Path) -> None: + add_provider_endpoint( + "openrouter", + label="primary", + api_key="k1", + api_base="https://old.example.com", + extra_headers={"X-Old": "1"}, + config_path=cfg_path, + ) + endpoints = add_provider_endpoint("openrouter", label="primary", api_key="k2", config_path=cfg_path) + + # A field omitted on the replacement is gone, not carried over from the old + # entry: this is a replace, not a merge. + assert len(endpoints) == 1 + assert endpoints[0].api_key == "k2" + assert endpoints[0].api_base is None + assert endpoints[0].extra_headers is None + + +def test_add_provider_endpoint_with_api_base_and_headers(cfg_path: Path) -> None: + endpoints = add_provider_endpoint( + "openrouter", + label="eu", + api_key="k1", + api_base="https://eu.example.com", + extra_headers={"X-Region": "eu"}, + config_path=cfg_path, + ) + + assert endpoints[0].api_base == "https://eu.example.com" + assert endpoints[0].extra_headers == {"X-Region": "eu"} + + +def test_add_provider_endpoint_unknown_provider_raises(cfg_path: Path) -> None: + with pytest.raises(KeyError): + add_provider_endpoint("nonexistent_provider", label="x", api_key="k", config_path=cfg_path) + + +def test_remove_provider_endpoint(cfg_path: Path) -> None: + add_provider_endpoint("openrouter", label="primary", api_key="k1", config_path=cfg_path) + add_provider_endpoint("openrouter", label="backup", api_key="k2", config_path=cfg_path) + + endpoints = remove_provider_endpoint("openrouter", "primary", config_path=cfg_path) + + assert [e.label for e in endpoints] == ["backup"] + section = _read(cfg_path)["providers"]["openrouter"] + assert [e["label"] for e in section["endpoints"]] == ["backup"] + + +def test_remove_absent_endpoint_is_noop(cfg_path: Path) -> None: + add_provider_endpoint("openrouter", label="primary", api_key="k1", config_path=cfg_path) + + endpoints = remove_provider_endpoint("openrouter", "not-there", config_path=cfg_path) + + assert [e.label for e in endpoints] == ["primary"] + + +def test_remove_provider_endpoint_unknown_provider_raises(cfg_path: Path) -> None: + with pytest.raises(KeyError): + remove_provider_endpoint("nonexistent_provider", "x", config_path=cfg_path) + + +def test_list_provider_endpoints_redacts_api_key(cfg_path: Path) -> None: + add_provider_endpoint( + "openrouter", + label="primary", + api_key="k1", + api_base="https://example.com", + config_path=cfg_path, + ) + + out = list_provider_endpoints("openrouter", config_path=cfg_path) + + assert out == [ + {"label": "primary", "api_key": "****set****", "api_base": "https://example.com", "extra_headers": None} + ] + + +def test_list_provider_endpoints_reports_empty_key(cfg_path: Path) -> None: + add_provider_endpoint("openrouter", label="primary", api_key="", config_path=cfg_path) + + out = list_provider_endpoints("openrouter", config_path=cfg_path) + + assert out[0]["api_key"] == "(empty)" + + +def test_list_provider_endpoints_default_when_none_configured(cfg_path: Path) -> None: + assert list_provider_endpoints("openrouter", config_path=cfg_path) == [] + + +def test_list_provider_endpoints_unknown_provider_raises(cfg_path: Path) -> None: + with pytest.raises(KeyError): + list_provider_endpoints("nonexistent_provider", config_path=cfg_path) + + def test_malformed_config_refuses_write_and_preserves_file(cfg_path: Path) -> None: # REGRESSION: a present-but-unparseable config must NOT be clobbered. from raven.config.loader import ConfigReadError From c2f53c9a654a0c577650e5347727a20bcbfd04c1 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 01:21:44 +0800 Subject: [PATCH 28/78] feat(*): manage provider endpoints from the tui Last stage of multi-endpoint support. Three RPC methods (model.endpoints / add_endpoint / remove_endpoint) follow the add-one-return-the-refreshed-list shape, keys redacted at the RPC boundary; the model picker gains an endpoints stage with the same list interaction its model list settled on (a to add, d to delete, Esc back); the session footer names the active endpoint when a provider has several, fed by the rotor's active_endpoint_label -- read without advancing the rotation cursor. Writes land in config and take effect on the next provider build, the same semantics save_key already has. The endpoint field lives on SessionInfo itself; the openrpc contract, generated types and hand-written mirror all moved in the same step. Co-authored-by: Claude (claude-fable-5) --- raven/providers/endpoint_rotor.py | 29 +- raven/tui_rpc/methods/__init__.py | 3 +- raven/tui_rpc/methods/model.py | 61 +++- raven/tui_rpc/methods/session.py | 4 + raven/tui_rpc/models.py | 44 +++ tests/test_provider_endpoint_rotor.py | 36 +++ tests/test_tui_rpc_model.py | 100 +++++- tests/test_tui_rpc_session_init_bundle.py | 53 ++- ui-tui/rpc-schema/openrpc.json | 138 ++++++++ ui-tui/src/__tests__/modelPicker.test.tsx | 130 ++++++++ ui-tui/src/components/branding.tsx | 10 +- ui-tui/src/components/modelPicker.tsx | 378 +++++++++++++++++++++- ui-tui/src/gatewayTypes.ts | 14 + ui-tui/src/rpc/generated.ts | 66 ++++ ui-tui/src/types.ts | 3 + 15 files changed, 1050 insertions(+), 19 deletions(-) diff --git a/raven/providers/endpoint_rotor.py b/raven/providers/endpoint_rotor.py index dc03061d..fb1fae68 100644 --- a/raven/providers/endpoint_rotor.py +++ b/raven/providers/endpoint_rotor.py @@ -134,17 +134,36 @@ def _healthy_order(self) -> list[int]: somewhere, and cooldown is a preference between healthy endpoints, not a breaker that can leave nothing to try. """ - n = len(self._inners) - now = time.monotonic() if self.strategy == "round_robin": start = self._state.index - self._state.index = (start + 1) % n - order = [(start + i) % n for i in range(n)] + self._state.index = (start + 1) % len(self._inners) else: - order = list(range(n)) + start = 0 + return self._order_from(start) + + def _order_from(self, start: int) -> list[int]: + """The order ``_healthy_order`` returns for a given starting index. + + Split out so the cursor advance stays in ``_healthy_order`` alone and + ``active_endpoint_label`` can ask the same question without answering + it differently or moving the rotation on. + """ + n = len(self._inners) + now = time.monotonic() + order = [(start + i) % n for i in range(n)] healthy = [i for i in order if not self._state.is_cooling(i, now)] return healthy or order + @property + def active_endpoint_label(self) -> str: + """Label of the endpoint the next request would go to. + + Read-only: unlike ``_healthy_order`` it never advances the round-robin + cursor, so asking is not a rotation. + """ + start = self._state.index if self.strategy == "round_robin" else 0 + return self._endpoints[self._order_from(start)[0]].label + async def _chat_attempt_with_retry( self, *, diff --git a/raven/tui_rpc/methods/__init__.py b/raven/tui_rpc/methods/__init__.py index 3b503523..5e85bc4e 100644 --- a/raven/tui_rpc/methods/__init__.py +++ b/raven/tui_rpc/methods/__init__.py @@ -126,7 +126,8 @@ def register_aligned_methods_except_system( register_session_methods(dispatcher, agent_loop_factory=agent_loop_factory) register_terminal_methods(dispatcher) register_stub_methods(dispatcher) - # model.{options,save_key,disconnect,add_model,remove_model}: real handlers + # model.{options,save_key,disconnect,add_model,remove_model,endpoints, + # add_endpoint,remove_endpoint}: real handlers # must come AFTER register_stub_methods (Dispatcher.register raises on # duplicate; the stub group no longer owns these names). register_model_methods(dispatcher) diff --git a/raven/tui_rpc/methods/model.py b/raven/tui_rpc/methods/model.py index 70652a1a..11d917eb 100644 --- a/raven/tui_rpc/methods/model.py +++ b/raven/tui_rpc/methods/model.py @@ -1,12 +1,15 @@ """``model.*`` RPC handlers — backend for the TUI ``/model`` v1 picker. -Five methods drive the picker: +Eight methods drive the picker: * ``model.options`` — current model/provider + one row per provider. * ``model.save_key`` — store an api_key (+ optional api_base) for a provider. * ``model.disconnect`` — clear a provider's stored credentials. * ``model.add_model`` / ``model.remove_model`` — edit a provider's curated model list. +* ``model.endpoints`` / ``model.add_endpoint`` / ``model.remove_endpoint`` — + edit the several url/key groups one provider section can carry, each write + answering with the refreshed (key-redacted) list. All write helpers live in ``raven.config.update_providers`` (the single write path for provider config); the handlers wrap the synchronous calls in @@ -23,9 +26,12 @@ from pydantic import ValidationError from raven.config.update_providers import ( + add_provider_endpoint, add_provider_model, get_provider_config, + list_provider_endpoints, list_providers, + remove_provider_endpoint, remove_provider_model, reset_provider, set_provider_fields, @@ -47,9 +53,12 @@ NotSupportedInV01Error, ) from raven.tui_rpc.models import ( + ModelAddEndpointParams, ModelAddModelParams, ModelDisconnectParams, + ModelEndpointsParams, ModelOptionsParams, + ModelRemoveEndpointParams, ModelRemoveModelParams, ModelSaveKeyParams, ) @@ -355,13 +364,58 @@ async def model_remove_model(params: dict) -> dict: } +async def _endpoints_off_loop(slug: str) -> list[dict[str, Any]]: + """The provider's endpoint list, api_key redacted, off the event loop.""" + try: + return await asyncio.to_thread(list_provider_endpoints, slug) + except KeyError as exc: + raise ConfigValidationError(str(exc), data={"slug": slug}) from exc + + +async def model_endpoints(params: dict) -> dict: + parsed = _parse(ModelEndpointsParams, params) + return {"endpoints": await _endpoints_off_loop(parsed.slug)} + + +async def model_add_endpoint(params: dict) -> dict: + parsed = _parse(ModelAddEndpointParams, params) + try: + # extra_headers is deliberately not a parameter: the picker has no screen + # that could collect one, and a field only `raven provider` can write is + # not made reachable by declaring it here. + await asyncio.to_thread( + add_provider_endpoint, + parsed.slug, + label=parsed.label, + api_key=parsed.api_key, + api_base=parsed.api_base, + ) + except KeyError as exc: + raise ConfigValidationError(str(exc), data={"slug": parsed.slug}) from exc + # Re-read rather than redacting what the write returned, so the one place + # deciding how a key is masked stays ``list_provider_endpoints``. + return {"endpoints": await _endpoints_off_loop(parsed.slug)} + + +async def model_remove_endpoint(params: dict) -> dict: + parsed = _parse(ModelRemoveEndpointParams, params) + try: + await asyncio.to_thread(remove_provider_endpoint, parsed.slug, parsed.label) + except KeyError as exc: + raise ConfigValidationError(str(exc), data={"slug": parsed.slug}) from exc + return {"endpoints": await _endpoints_off_loop(parsed.slug)} + + def register_model_methods(dispatcher: "Dispatcher") -> None: - """Register the five ``model.*`` handlers on a dispatcher instance.""" + """Register the eight ``model.*`` handlers on a dispatcher instance.""" dispatcher.register("model.options", model_options) dispatcher.register("model.save_key", model_save_key) dispatcher.register("model.disconnect", model_disconnect) dispatcher.register("model.add_model", model_add_model) dispatcher.register("model.remove_model", model_remove_model) + dispatcher.register("model.endpoints", model_endpoints) + dispatcher.register("model.add_endpoint", model_add_endpoint) + dispatcher.register("model.remove_endpoint", model_remove_endpoint) __all__ = [ @@ -370,6 +424,9 @@ def register_model_methods(dispatcher: "Dispatcher") -> None: "model_disconnect", "model_add_model", "model_remove_model", + "model_endpoints", + "model_add_endpoint", + "model_remove_endpoint", "register_model_methods", "_build_provider_entry", ] diff --git a/raven/tui_rpc/methods/session.py b/raven/tui_rpc/methods/session.py index 459f4a0c..106a9704 100644 --- a/raven/tui_rpc/methods/session.py +++ b/raven/tui_rpc/methods/session.py @@ -157,6 +157,10 @@ def _default_session_info( "version": _RAVEN_VERSION, "cwd": os.getcwd(), "mcp_servers": [], + # Which of a multi-endpoint provider's endpoints this session is on. + # None for every single-endpoint provider -- there is one address and it + # carries no label worth showing. + "endpoint": getattr(getattr(agent_loop, "provider", None), "active_endpoint_label", None), } # Nudge the status bar to run `raven upgrade` when the cached latest release diff --git a/raven/tui_rpc/models.py b/raven/tui_rpc/models.py index 7e62d3d9..651fe8ac 100644 --- a/raven/tui_rpc/models.py +++ b/raven/tui_rpc/models.py @@ -608,6 +608,46 @@ class ModelRemoveModelResult(_Strict): provider: ModelOptionProvider +class ProviderEndpointInfo(_Strict): + """One of a provider section's endpoints, as the picker shows it.""" + + label: str + api_key: str = Field(..., description="Redacted for display: `****set****` or `(empty)`.") + api_base: str | None = None + extra_headers: dict[str, str] | None = None + + +class ModelEndpointsParams(_Strict): + slug: str + session_id: str | None = None + + +class ModelEndpointsResult(_Strict): + endpoints: list[ProviderEndpointInfo] + + +class ModelAddEndpointParams(_Strict): + slug: str + label: str = Field(..., description="Idempotency key: an existing entry with this label is replaced wholesale.") + api_key: str = "" + api_base: str | None = None + session_id: str | None = None + + +class ModelAddEndpointResult(_Strict): + endpoints: list[ProviderEndpointInfo] + + +class ModelRemoveEndpointParams(_Strict): + slug: str + label: str + session_id: str | None = None + + +class ModelRemoveEndpointResult(_Strict): + endpoints: list[ProviderEndpointInfo] + + # --------------------------------------------------------------------------- # config.* methods # --------------------------------------------------------------------------- @@ -881,6 +921,9 @@ class ToolsConfigureParams(_Strict): "model.disconnect": (ModelDisconnectParams, ModelDisconnectResult), "model.add_model": (ModelAddModelParams, ModelAddModelResult), "model.remove_model": (ModelRemoveModelParams, ModelRemoveModelResult), + "model.endpoints": (ModelEndpointsParams, ModelEndpointsResult), + "model.add_endpoint": (ModelAddEndpointParams, ModelAddEndpointResult), + "model.remove_endpoint": (ModelRemoveEndpointParams, ModelRemoveEndpointResult), # config.* "config.get": (ConfigGetParams, ConfigGetResult), "config.set": (ConfigSetParams, ConfigSetResult), @@ -915,6 +958,7 @@ class ToolsConfigureParams(_Strict): "McpToolInfo", "SkillInfo", "ModelOptionProvider", + "ProviderEndpointInfo", "UsageSnapshot", "CliResult", "StubResult", diff --git a/tests/test_provider_endpoint_rotor.py b/tests/test_provider_endpoint_rotor.py index a19075cb..47046421 100644 --- a/tests/test_provider_endpoint_rotor.py +++ b/tests/test_provider_endpoint_rotor.py @@ -142,6 +142,42 @@ async def test_round_robin_cursor_advances_each_call(clock): assert (e0.chat_calls, e1.chat_calls, e2.chat_calls) == (2, 1, 1) +async def test_active_endpoint_label_names_the_next_endpoint_without_rotating(clock): + """The banner reads this to say which account is answering. Asking is not a + request, so it must not consume a round-robin slot -- a getter that advanced + the cursor would skip an endpoint on every render.""" + e0 = _StubInner("e0") + e1 = _StubInner("e1") + rotor = _make_rotor([e0, e1], strategy="round_robin") + + assert [rotor.active_endpoint_label for _ in range(3)] == ["e0", "e0", "e0"] + + await rotor.chat_with_retry(messages=[], model="m") + + assert rotor.active_endpoint_label == "e1" + + +async def test_active_endpoint_label_skips_a_cooling_endpoint(clock): + """It names where the next request would land, which under sticky is the + first endpoint that is not cooling -- not simply the first one.""" + e0 = _StubInner( + "e0", + chat_script=[ + LLMResponse(content="e0 unavailable", finish_reason="error", error_classification=_FALLBACK_FATAL) + ], + ) + e1 = _StubInner("e1") + rotor = _make_rotor([e0, e1], strategy="sticky") + + assert rotor.active_endpoint_label == "e0" + + await rotor.chat_with_retry(messages=[], model="m") + assert rotor.active_endpoint_label == "e1" + + clock.now += 30.0 + assert rotor.active_endpoint_label == "e0" + + async def test_non_fallback_error_returns_immediately_without_rotating(clock): e0 = _StubInner( "e0", diff --git a/tests/test_tui_rpc_model.py b/tests/test_tui_rpc_model.py index 0e689c76..99d30873 100644 --- a/tests/test_tui_rpc_model.py +++ b/tests/test_tui_rpc_model.py @@ -1,6 +1,6 @@ """Tests for the ``model.*`` RPC handlers (TUI ``/model`` v1 backend). -The five handlers wrap ``raven.config.update_providers`` write/read helpers +The eight handlers wrap ``raven.config.update_providers`` write/read helpers plus the provider registry. Config is sandboxed by redirecting ``Path.home()`` to a tmp dir (same mechanism as ``test_tui_rpc_config`` / ``test_tui_rpc_setup``) so the real user config is never touched. No network is hit. @@ -18,9 +18,12 @@ from raven.tui_rpc.errors import ConfigValidationError, NotSupportedInV01Error from raven.tui_rpc.methods import model as model_module from raven.tui_rpc.methods.model import ( + model_add_endpoint, model_add_model, model_disconnect, + model_endpoints, model_options, + model_remove_endpoint, model_remove_model, model_save_key, ) @@ -252,6 +255,90 @@ async def test_add_model_unknown_provider_rejected(fake_home: Path) -> None: await model_add_model({"slug": "no_such_provider", "model": "x"}) +# ---------------------------------------------------------------------------- +# model.endpoints / model.add_endpoint / model.remove_endpoint +# ---------------------------------------------------------------------------- + + +async def test_add_endpoint_answers_with_the_refreshed_list(fake_home: Path) -> None: + result = await model_add_endpoint( + {"slug": "deepseek", "label": "eu", "api_key": "sk-eu", "api_base": "https://eu.example.test/v1"} + ) + assert [ep["label"] for ep in result["endpoints"]] == ["eu"] + assert result["endpoints"][0]["api_base"] == "https://eu.example.test/v1" + + listed = await model_endpoints({"slug": "deepseek"}) + assert listed == result + + +async def test_endpoints_never_hand_back_the_key(fake_home: Path) -> None: + """The picker only ever displays this list, and a key it did not need to see + is a key a screenshot can leak.""" + await model_add_endpoint({"slug": "deepseek", "label": "eu", "api_key": "sk-eu-secret"}) + await model_add_endpoint({"slug": "deepseek", "label": "keyless"}) + + by_label = {ep["label"]: ep["api_key"] for ep in (await model_endpoints({"slug": "deepseek"}))["endpoints"]} + + assert "sk-eu-secret" not in by_label.values() + assert by_label == {"eu": "****set****", "keyless": "(empty)"} + + +async def test_add_endpoint_replaces_the_entry_with_the_same_label(fake_home: Path) -> None: + """``label`` is the idempotency key, so re-adding it is how a rotated key is + written -- appending a second entry would leave the dead key in rotation.""" + await model_add_endpoint({"slug": "deepseek", "label": "eu", "api_key": "sk-old"}) + result = await model_add_endpoint( + {"slug": "deepseek", "label": "eu", "api_key": "sk-new", "api_base": "https://eu.example.test/v1"} + ) + + assert [ep["label"] for ep in result["endpoints"]] == ["eu"] + assert result["endpoints"][0]["api_base"] == "https://eu.example.test/v1" + section = json.loads((fake_home / ".raven" / "config.json").read_text())["providers"]["deepseek"] + assert [ep["apiKey"] for ep in section["endpoints"]] == ["sk-new"] + + +async def test_remove_endpoint_reflected_in_the_list(fake_home: Path) -> None: + await model_add_endpoint({"slug": "deepseek", "label": "eu", "api_key": "sk-eu"}) + await model_add_endpoint({"slug": "deepseek", "label": "us", "api_key": "sk-us"}) + + result = await model_remove_endpoint({"slug": "deepseek", "label": "eu"}) + + assert [ep["label"] for ep in result["endpoints"]] == ["us"] + listed = await model_endpoints({"slug": "deepseek"}) + assert [ep["label"] for ep in listed["endpoints"]] == ["us"] + + +async def test_removing_an_absent_label_is_a_no_op(fake_home: Path) -> None: + await model_add_endpoint({"slug": "deepseek", "label": "eu", "api_key": "sk-eu"}) + + result = await model_remove_endpoint({"slug": "deepseek", "label": "never-existed"}) + + assert [ep["label"] for ep in result["endpoints"]] == ["eu"] + + +@pytest.mark.parametrize( + "call", + [ + pytest.param(lambda: model_endpoints({"slug": "no_such_provider"}), id="endpoints"), + pytest.param(lambda: model_add_endpoint({"slug": "no_such_provider", "label": "eu"}), id="add_endpoint"), + pytest.param(lambda: model_remove_endpoint({"slug": "no_such_provider", "label": "eu"}), id="remove_endpoint"), + ], +) +async def test_endpoint_handlers_reject_an_unknown_provider(fake_home: Path, call) -> None: + with pytest.raises(ConfigValidationError): + await call() + + +async def test_endpoint_handlers_accept_session_id(fake_home: Path) -> None: + # The picker passes its session down like it does for every other model.* + # call; a strict param model would reject the key otherwise. + await model_add_endpoint({"slug": "deepseek", "label": "eu", "session_id": "tui:default"}) + await model_endpoints({"slug": "deepseek", "session_id": "tui:default"}) + result = await model_remove_endpoint({"slug": "deepseek", "label": "eu", "session_id": "tui:default"}) + + assert result["endpoints"] == [] + + # ---------------------------------------------------------------------------- # Dispatcher wiring # ---------------------------------------------------------------------------- @@ -278,6 +365,17 @@ async def test_model_methods_registered_via_helper(fake_home: Path) -> None: ) assert resp["error"]["code"] == -32012 + resp = await d.dispatch( + { + "jsonrpc": "2.0", + "id": 3, + "method": "model.endpoints", + "params": {"slug": "deepseek"}, + } + ) + assert "error" not in resp + assert resp["result"] == {"endpoints": []} + # ---------------------------------------------------------------------------- # Regressions (code review) diff --git a/tests/test_tui_rpc_session_init_bundle.py b/tests/test_tui_rpc_session_init_bundle.py index a56c05db..2d3c052e 100644 --- a/tests/test_tui_rpc_session_init_bundle.py +++ b/tests/test_tui_rpc_session_init_bundle.py @@ -267,7 +267,7 @@ def test_resolve_context_window_helper_removed() -> None: def test_default_session_info_key_set_matches_expected_v030(fake_agent_loop, config) -> None: - """wire-shape lock — info dict has exactly the 11 expected keys. + """wire-shape lock — info dict has exactly the 12 expected keys. Anti-drift gate: adding a new field to the init bundle MUST update this expected set, forcing an explicit spec amendment, until the dict is @@ -289,6 +289,7 @@ def test_default_session_info_key_set_matches_expected_v030(fake_agent_loop, con "lazy", # extended bundle "usage", + "endpoint", } assert set(info) == expected_keys, ( f"init bundle key set drift: unexpected={set(info) - expected_keys}, missing={expected_keys - set(info)}" @@ -383,3 +384,53 @@ def test_default_session_info_omits_the_nudge_when_up_to_date(fake_agent_loop, c assert "update_available" not in info assert "update_command" not in info + + +# --------------------------------------------------------------------------- +# Which endpoint the session is on (multi-endpoint providers only) +# --------------------------------------------------------------------------- + + +def _rotor(labels: list[str], strategy: str = "sticky"): + from raven.providers.base import LLMProvider + from raven.providers.endpoint_rotor import EndpointRotorProvider + from raven.providers.endpoints import ResolvedEndpoint + + class _Inner(LLMProvider): + async def chat(self, messages, tools=None, model=None, **kwargs): # pragma: no cover - never called + raise AssertionError("the banner must not send a request") + + def get_default_model(self) -> str: + return "m" + + return EndpointRotorProvider( + endpoints=[ResolvedEndpoint(label=label, api_key="k", api_base=None, extra_headers=None) for label in labels], + make_inner=lambda _e: _Inner(api_key="test"), + default_model="m", + strategy=strategy, + ) + + +def test_default_session_info_names_the_endpoint_in_use(config) -> None: + """A rotor serves several accounts, so which one is answering is a fact the + banner has to carry -- naming only the provider makes them indistinguishable.""" + loop = _FakeAgentLoop(with_usage_tracker=True) + loop.provider = _rotor(["eu", "us"]) + + assert _default_session_info(loop, config)["endpoint"] == "eu" + + +def test_default_session_info_endpoint_is_none_for_a_single_endpoint_provider(fake_agent_loop, config) -> None: + """Every provider but the rotor is reached at one address with no label, so + the field is present-and-null rather than a borrowed name.""" + assert _default_session_info(fake_agent_loop, config)["endpoint"] is None + + +def test_reading_the_banner_endpoint_does_not_rotate(config) -> None: + """Under round_robin the order cursor advances per request. Building the + banner is not a request, and a getter that moved it would skip an endpoint + every time the panel was rendered.""" + loop = _FakeAgentLoop(with_usage_tracker=True) + loop.provider = _rotor(["eu", "us"], strategy="round_robin") + + assert [_default_session_info(loop, config)["endpoint"] for _ in range(3)] == ["eu", "eu", "eu"] diff --git a/ui-tui/rpc-schema/openrpc.json b/ui-tui/rpc-schema/openrpc.json index 06df7a25..71eb4470 100644 --- a/ui-tui/rpc-schema/openrpc.json +++ b/ui-tui/rpc-schema/openrpc.json @@ -838,6 +838,126 @@ { "$ref": "#/components/errors/ConfigValidationError" } ] }, + { + "name": "model.endpoints", + "summary": "List a provider's endpoints, api_key redacted for display.", + "params": [ + { + "name": "slug", + "required": true, + "schema": { "type": "string" } + }, + { + "name": "session_id", + "required": false, + "schema": { "type": "string" } + } + ], + "result": { + "name": "ModelEndpointsResult", + "schema": { + "type": "object", + "additionalProperties": false, + "required": ["endpoints"], + "properties": { + "endpoints": { + "type": "array", + "items": { "$ref": "#/components/schemas/ProviderEndpointInfo" } + } + } + } + }, + "errors": [ + { "$ref": "#/components/errors/ConfigValidationError" } + ] + }, + { + "name": "model.add_endpoint", + "summary": "Add or replace one of a provider's endpoints, keyed by label.", + "params": [ + { + "name": "slug", + "required": true, + "schema": { "type": "string" } + }, + { + "name": "label", + "description": "Idempotency key: an existing entry with this label is replaced wholesale.", + "required": true, + "schema": { "type": "string" } + }, + { + "name": "api_key", + "required": false, + "schema": { "type": "string" } + }, + { + "name": "api_base", + "required": false, + "schema": { "type": "string" } + }, + { + "name": "session_id", + "required": false, + "schema": { "type": "string" } + } + ], + "result": { + "name": "ModelAddEndpointResult", + "schema": { + "type": "object", + "additionalProperties": false, + "required": ["endpoints"], + "properties": { + "endpoints": { + "type": "array", + "items": { "$ref": "#/components/schemas/ProviderEndpointInfo" } + } + } + } + }, + "errors": [ + { "$ref": "#/components/errors/ConfigValidationError" } + ] + }, + { + "name": "model.remove_endpoint", + "summary": "Remove one of a provider's endpoints by label.", + "params": [ + { + "name": "slug", + "required": true, + "schema": { "type": "string" } + }, + { + "name": "label", + "required": true, + "schema": { "type": "string" } + }, + { + "name": "session_id", + "required": false, + "schema": { "type": "string" } + } + ], + "result": { + "name": "ModelRemoveEndpointResult", + "schema": { + "type": "object", + "additionalProperties": false, + "required": ["endpoints"], + "properties": { + "endpoints": { + "type": "array", + "items": { "$ref": "#/components/schemas/ProviderEndpointInfo" } + } + } + } + }, + "errors": [ + { "$ref": "#/components/errors/ConfigValidationError" } + ] + }, { "name": "config.get", "summary": "Read hot-changeable config fields (whitelist).", @@ -1431,6 +1551,24 @@ "description": { "type": "string" } } }, + "ProviderEndpointInfo": { + "type": "object", + "additionalProperties": false, + "description": "One of a provider section's several url/key groups. `label` is the idempotency key the write methods address an entry by.", + "required": ["label", "api_key"], + "properties": { + "label": { "type": "string" }, + "api_key": { + "type": "string", + "description": "Redacted for display: `****set****` or `(empty)`." + }, + "api_base": { "type": "string" }, + "extra_headers": { + "type": "object", + "additionalProperties": { "type": "string" } + } + } + }, "UsageSnapshot": { "type": "object", "additionalProperties": false, diff --git a/ui-tui/src/__tests__/modelPicker.test.tsx b/ui-tui/src/__tests__/modelPicker.test.tsx index ab119c9b..7654fa01 100644 --- a/ui-tui/src/__tests__/modelPicker.test.tsx +++ b/ui-tui/src/__tests__/modelPicker.test.tsx @@ -641,6 +641,136 @@ describe('ModelPicker', () => { h.unmount() }) + it('lists a provider endpoints, masked, on `e`', async () => { + const h = mount([anthropic], method => { + if (method === 'model.endpoints') { + return { + endpoints: [ + { api_base: 'https://eu.example.test/v1', api_key: '****set****', label: 'eu' }, + { api_base: null, api_key: '(empty)', label: 'spare' } + ] + } + } + + return {} + }) + await delay(60) + + await h.type(ENTER) + await waitForFrame(h, 'step 2/2') + + await h.type('e') + await waitForFrame(h, 'eu · ****set**** · https://eu.example.test/v1') + + expect(h.gw.request).toHaveBeenCalledWith('model.endpoints', expect.objectContaining({ slug: 'anthropic' })) + // The second row proves the empty-key spelling renders as its own state + // rather than collapsing into a blank cell. + expect(h.frame()).toContain('spare · (empty)') + + h.unmount() + }) + + it('adds an endpoint via model.add_endpoint with all three fields', async () => { + const h = mount([anthropic], (method, params) => { + if (method === 'model.endpoints') { + return { endpoints: [] } + } + + if (method === 'model.add_endpoint') { + return { endpoints: [{ api_base: params.api_base, api_key: '****set****', label: params.label }] } + } + + return {} + }) + await delay(60) + + await h.type(ENTER) + await waitForFrame(h, 'step 2/2') + await h.type('e') + await waitForFrame(h, 'no endpoints') + + // label -> Enter -> key -> Enter -> base -> Enter submits. + await h.type('a') + await h.type('eu') + await h.type(ENTER) + await h.type('sk-eu') + await h.type(ENTER) + await h.type('https://eu.example.test/v1') + await h.type(ENTER) + + expect(h.gw.request).toHaveBeenCalledWith( + 'model.add_endpoint', + expect.objectContaining({ + api_base: 'https://eu.example.test/v1', + api_key: 'sk-eu', + label: 'eu', + slug: 'anthropic' + }) + ) + // The write answers with the refreshed list, and the screen shows it. + await waitForFrame(h, 'eu · ****set****') + + h.unmount() + }) + + it('removes the selected endpoint via model.remove_endpoint', async () => { + const h = mount([anthropic], method => { + if (method === 'model.endpoints') { + return { + endpoints: [ + { api_base: null, api_key: '****set****', label: 'eu' }, + { api_base: null, api_key: '****set****', label: 'us' } + ] + } + } + + if (method === 'model.remove_endpoint') { + return { endpoints: [{ api_base: null, api_key: '****set****', label: 'us' }] } + } + + return {} + }) + await delay(60) + + await h.type(ENTER) + await waitForFrame(h, 'step 2/2') + await h.type('e') + await waitForFrame(h, 'eu · ****set****') + + await h.type(DOWN) + await h.type('d') + + expect(h.gw.request).toHaveBeenCalledWith( + 'model.remove_endpoint', + expect.objectContaining({ label: 'us', slug: 'anthropic' }) + ) + + h.unmount() + }) + + it('returns from the endpoint list to the model list on Esc', async () => { + const h = mount([anthropic], method => (method === 'model.endpoints' ? { endpoints: [] } : {})) + await delay(60) + + await h.type(ENTER) + await waitForFrame(h, 'step 2/2') + await h.type('e') + await waitForFrame(h, 'no endpoints') + + // Asserted by what the next Enter reaches rather than by screen text: + // `frame()` accumulates, so the model list is on screen either way. + await h.type(ESCAPE) + // Ink holds a lone ESC back to see whether it opens a sequence, so the next + // key has to arrive after that window or the two are read as one chord. + await delay(120) + await h.type(ENTER) + await delay(30) + + expect(h.onSelect).toHaveBeenCalledWith('claude-sonnet-4-6', 'anthropic') + + h.unmount() + }) + it('emits a structured model + provider selection on Enter', async () => { const h = mount([anthropic]) await delay(60) diff --git a/ui-tui/src/components/branding.tsx b/ui-tui/src/components/branding.tsx index 7b443aa6..19e0b969 100644 --- a/ui-tui/src/components/branding.tsx +++ b/ui-tui/src/components/branding.tsx @@ -230,10 +230,12 @@ export function SessionPanel({ info, maxCols, sid, t }: SessionPanelProps) { const lineBudget = Math.max(12, w - 2) const strip = (s: string) => (s.endsWith('_tools') ? s.slice(0, -6) : s) - // Footer meta (model · provider · session). Kept beside `/help` only when it - // fits the column; otherwise the footer becomes a column so the whole meta - // line drops below `/help` instead of wrapping mid-string. - const footerMeta = `${info.model.split('/').pop()} · ${formatProvider(info.provider, info.model_id)}${sid ? ` · ${sid}` : ''}` + // Footer meta (model · provider · endpoint · session). Kept beside `/help` + // only when it fits the column; otherwise the footer becomes a column so the + // whole meta line drops below `/help` instead of wrapping mid-string. The + // endpoint segment appears only for a provider that has several, since a + // single-endpoint one has no label worth a slot. + const footerMeta = `${info.model.split('/').pop()} · ${formatProvider(info.provider, info.model_id)}${info.endpoint ? ` · ${info.endpoint}` : ''}${sid ? ` · ${sid}` : ''}` const footerInline = FOOTER_HELP_TEXT.length + 2 + footerMeta.length <= w // ── Local collapse state for each section ── diff --git a/ui-tui/src/components/modelPicker.tsx b/ui-tui/src/components/modelPicker.tsx index adeb9e4e..75687689 100644 --- a/ui-tui/src/components/modelPicker.tsx +++ b/ui-tui/src/components/modelPicker.tsx @@ -9,7 +9,12 @@ import { Box, Text, useInput, useStdout } from '@hermes/ink' import { useCallback, useEffect, useMemo, useState } from 'react' import type { GatewayClient } from '../gatewayClientStub.js' -import type { ModelOptionProvider, ModelOptionsResponse } from '../gatewayTypes.js' +import type { + ModelEndpointsResponse, + ModelOptionProvider, + ModelOptionsResponse, + ProviderEndpointInfo +} from '../gatewayTypes.js' import type { LaunchResult } from '../lib/externalCli.js' import type { Theme } from '../theme.js' @@ -21,7 +26,16 @@ const VISIBLE = 12 const MIN_WIDTH = 40 const MAX_WIDTH = 90 -type Stage = 'provider' | 'addProvider' | 'key' | 'model' | 'addModel' | 'disconnect' | 'oauthLogin' +type Stage = + | 'provider' + | 'addProvider' + | 'key' + | 'model' + | 'addModel' + | 'disconnect' + | 'oauthLogin' + | 'endpoints' + | 'addEndpoint' /** Where the sign-in handoff is: waiting to start, running, or back from it. */ type LoginPhase = 'idle' | 'running' | 'done' @@ -55,6 +69,11 @@ function unconfiguredWarning(p: ModelOptionProvider): string { } type KeyField = 'api_key' | 'api_base' +/** The three fields the add-endpoint screen collects, in the order it asks. */ +type EndpointField = 'label' | 'api_key' | 'api_base' + +const ENDPOINT_FIELD_ORDER: EndpointField[] = ['label', 'api_key', 'api_base'] + export function ModelPicker({ gw, launcher, onCancel, onSelect, sessionId, suspend, t }: ModelPickerProps) { const [providers, setProviders] = useState([]) const [currentModel, setCurrentModel] = useState('') @@ -70,6 +89,14 @@ export function ModelPicker({ gw, launcher, onCancel, onSelect, sessionId, suspe const [keySaving, setKeySaving] = useState(false) const [keyError, setKeyError] = useState('') const [modelNameInput, setModelNameInput] = useState('') + const [endpoints, setEndpoints] = useState([]) + const [endpointIdx, setEndpointIdx] = useState(0) + const [endpointField, setEndpointField] = useState('label') + const [endpointInputs, setEndpointInputs] = useState>({ + api_base: '', + api_key: '', + label: '' + }) // The sign-in screen names its provider from here rather than from the // selection: a successful login moves that provider out of the unconfigured // list the cursor is pointing into, which would rename the screen mid-flow. @@ -157,7 +184,44 @@ export function ModelPicker({ gw, launcher, onCancel, onSelect, sessionId, suspe : provider?.authenticated === false const names = useMemo(() => providerDisplayNames(rowsForStage), [rowsForStage]) + // Refetch only, same as ``loadOptions``: the endpoint list is not carried by + // ``model.options``, so every screen that shows it asks for it. + const loadEndpoints = useCallback( + async (slug: string) => { + try { + const raw = await gw.request('model.endpoints', { + slug, + ...(sessionId ? { session_id: sessionId } : {}) + }) + setEndpoints(asRpcResult(raw)?.endpoints ?? []) + } catch (e: unknown) { + setKeyError(rpcErrorMessage(e)) + } + }, + [gw, sessionId] + ) + + const clearEndpointInputs = () => { + setEndpointInputs({ api_base: '', api_key: '', label: '' }) + setEndpointField('label') + } + const back = () => { + if (stage === 'addEndpoint') { + setStage('endpoints') + clearEndpointInputs() + setKeyError('') + + return + } + + if (stage === 'endpoints') { + setStage('model') + setKeyError('') + + return + } + if (stage === 'addProvider') { setStage('provider') setFromAddList(false) @@ -268,7 +332,7 @@ export function ModelPicker({ gw, launcher, onCancel, onSelect, sessionId, suspe // Both stages type into a field, and the sign-in has handed the terminal to // a child process -- leaving the screen from under either one loses input // the user has already given. - closeOnQ: stage !== 'key' && stage !== 'addModel', + closeOnQ: stage !== 'key' && stage !== 'addModel' && stage !== 'addEndpoint', disabled: loginPhase === 'running', onBack: back, onClose: onCancel @@ -455,6 +519,164 @@ export function ModelPicker({ gw, launcher, onCancel, onSelect, sessionId, suspe return } + // Add-endpoint sub-input: label, then key, then base. + if (stage === 'addEndpoint') { + if (keySaving) { + return + } + + const fieldIdx = ENDPOINT_FIELD_ORDER.indexOf(endpointField) + + if (key.tab) { + setEndpointField(ENDPOINT_FIELD_ORDER[(fieldIdx + 1) % ENDPOINT_FIELD_ORDER.length]) + + return + } + + if (key.return) { + // Enter advances through the fields and only submits on the last one, + // so the whole endpoint can be entered without reaching for Tab. + if (fieldIdx < ENDPOINT_FIELD_ORDER.length - 1) { + setEndpointField(ENDPOINT_FIELD_ORDER[fieldIdx + 1]) + + return + } + + const label = endpointInputs.label.trim() + + if (!provider) { + return + } + + // The label is what every later edit addresses this entry by, so an + // unnamed one is not something the list could offer back. + if (!label) { + setKeyError('a label is required') + + return + } + + const apiKey = endpointInputs.api_key.trim() + const apiBase = endpointInputs.api_base.trim() + + setKeySaving(true) + setKeyError('') + gw.request('model.add_endpoint', { + slug: provider.slug, + label, + ...(apiKey ? { api_key: apiKey } : {}), + ...(apiBase ? { api_base: apiBase } : {}), + ...(sessionId ? { session_id: sessionId } : {}) + }) + .then(raw => { + const r = asRpcResult(raw) + + if (!r?.endpoints) { + setKeyError('failed to add endpoint') + setKeySaving(false) + + return + } + + setEndpoints(r.endpoints) + setEndpointIdx( + Math.max( + 0, + r.endpoints.findIndex(ep => ep.label === label) + ) + ) + clearEndpointInputs() + setKeySaving(false) + setStage('endpoints') + }) + .catch((e: unknown) => { + setKeyError(rpcErrorMessage(e)) + setKeySaving(false) + }) + + return + } + + if (key.backspace || key.delete) { + setEndpointInputs(v => ({ ...v, [endpointField]: v[endpointField].slice(0, -1) })) + + return + } + + if (ch === '\u0015') { + setEndpointInputs(v => ({ ...v, [endpointField]: '' })) + + return + } + + if (ch && !key.ctrl && !key.meta) { + setEndpointInputs(v => ({ ...v, [endpointField]: v[endpointField] + ch })) + } + + return + } + + // Endpoint list stage: same add/delete vocabulary as the model list. + if (stage === 'endpoints') { + if (keySaving) { + return + } + + if (key.upArrow && endpointIdx > 0) { + setEndpointIdx(v => v - 1) + + return + } + + if (key.downArrow && endpointIdx < endpoints.length - 1) { + setEndpointIdx(v => v + 1) + + return + } + + if (ch.toLowerCase() === 'a') { + clearEndpointInputs() + setKeyError('') + setStage('addEndpoint') + + return + } + + if (ch.toLowerCase() === 'd' || ch.toLowerCase() === 'x') { + const target = endpoints[endpointIdx] + + if (!provider || !target) { + return + } + + setKeySaving(true) + setKeyError('') + gw.request('model.remove_endpoint', { + slug: provider.slug, + label: target.label, + ...(sessionId ? { session_id: sessionId } : {}) + }) + .then(raw => { + const r = asRpcResult(raw) + + if (r?.endpoints) { + setEndpoints(r.endpoints) + setEndpointIdx(idx => Math.max(0, Math.min(idx, r.endpoints!.length - 1))) + } + + setKeySaving(false) + }) + .catch((e: unknown) => { + setKeyError(rpcErrorMessage(e)) + setKeySaving(false) + }) + + return + } + + return + } + // Disconnect confirmation stage if (stage === 'disconnect') { if (ch.toLowerCase() === 'y' || key.return) { @@ -605,6 +827,17 @@ export function ModelPicker({ gw, launcher, onCancel, onSelect, sessionId, suspe return } + // Model stage: manage the several accounts/regions behind this provider. + if (ch.toLowerCase() === 'e' && stage === 'model' && provider && !keySaving) { + setEndpoints([]) + setEndpointIdx(0) + setKeyError('') + setStage('endpoints') + void loadEndpoints(provider.slug) + + return + } + // Model stage: delete the highlighted model name from the provider's list. if ((ch.toLowerCase() === 'd' || ch.toLowerCase() === 'x') && stage === 'model' && !keySaving) { const model = models[modelIdx] @@ -797,6 +1030,141 @@ export function ModelPicker({ gw, launcher, onCancel, onSelect, sessionId, suspe ) } + // ── Endpoint list stage ────────────────────────────────────────────── + if (stage === 'endpoints' && provider) { + const rows = endpoints.map( + ep => `${ep.label} · ${ep.api_key || '(empty)'}${ep.api_base ? ` · ${ep.api_base}` : ''}` + ) + const { items, offset } = windowItems(rows, endpointIdx, VISIBLE) + + return ( + + + Endpoints for {provider.name} + + + + Several accounts or regions under one provider · keys are never shown + + + + {keyError ? `error: ${keyError}` : ' '} + + + + {offset > 0 ? ` ↑ ${offset} more` : ' '} + + + {Array.from({ length: VISIBLE }, (_, i) => { + const row = items[i] + const idx = offset + i + + if (!row) { + return !rows.length && i === 0 ? ( + + no endpoints configured · a adds one + + ) : ( + + {' '} + + ) + } + + return ( + + {endpointIdx === idx ? '▸ ' : ' '} + {idx + 1}. {row} + + ) + })} + + + {offset + VISIBLE < rows.length ? ` ↓ ${rows.length - offset - VISIBLE} more` : ' '} + + + {keySaving ? ( + + saving… + + ) : ( + ↑/↓ select · a add · d/x delete · Esc back · q close + )} + + ) + } + + // ── Add endpoint stage ─────────────────────────────────────────────── + if (stage === 'addEndpoint' && provider) { + const fields: { label: string; name: EndpointField; value: string }[] = [ + { label: 'Label', name: 'label', value: endpointInputs.label }, + { label: 'API key', name: 'api_key', value: '•'.repeat(Math.min(endpointInputs.api_key.length, 40)) }, + { label: 'API base (optional)', name: 'api_base', value: endpointInputs.api_base } + ] + const caret = keySaving ? '' : '▎' + + return ( + + + Add endpoint to {provider.name} + + + + A label already in the list replaces that entry · Tab switches field + + + + {' '} + + + {fields.map(field => { + const focused = endpointField === field.name + + return ( + + + {focused ? '▸ ' : ' '} + {field.label}: + + + + {' '} + {field.value || '(empty)'} + {focused ? caret : ''} + + + ) + })} + + + {' '} + + + {keyError ? ( + + error: {keyError} + + ) : keySaving ? ( + + saving… + + ) : ( + + {' '} + + )} + + Enter next/add · Tab field · Ctrl+U clear · Esc back + + ) + } + // ── OAuth sign-in stage ────────────────────────────────────────────── if (stage === 'oauthLogin' && loginTarget) { const command = `raven provider login ${loginTarget.slug.replace(/_/g, '-')}` @@ -1048,8 +1416,8 @@ export function ModelPicker({ gw, launcher, onCancel, onSelect, sessionId, suspe {models.length - ? '↑/↓ select · Enter switch · a add · d/x delete · Esc back · q close' - : 'a add model · Enter/Esc back · q close'} + ? '↑/↓ select · Enter switch · a add · d/x delete · e endpoints · Esc back' + : 'a add model · e endpoints · Enter/Esc back · q close'} ) diff --git a/ui-tui/src/gatewayTypes.ts b/ui-tui/src/gatewayTypes.ts index 9e1d5648..2a37fbf8 100644 --- a/ui-tui/src/gatewayTypes.ts +++ b/ui-tui/src/gatewayTypes.ts @@ -362,6 +362,20 @@ export interface ModelOptionsResponse { providers?: ModelOptionProvider[] } +// One of the several url/key groups a provider section can carry. `api_key` +// arrives redacted (`****set****` / `(empty)`) — the gateway never sends the +// real one back, so there is nothing here to unmask. +export interface ProviderEndpointInfo { + api_base?: null | string + api_key?: string + extra_headers?: null | Record + label: string +} + +export interface ModelEndpointsResponse { + endpoints?: ProviderEndpointInfo[] +} + // ── MCP ────────────────────────────────────────────────────────────── export interface ReloadMcpResponse { diff --git a/ui-tui/src/rpc/generated.ts b/ui-tui/src/rpc/generated.ts index 9abe587c..36ffcfa8 100644 --- a/ui-tui/src/rpc/generated.ts +++ b/ui-tui/src/rpc/generated.ts @@ -169,6 +169,23 @@ export interface ModelLabel { label: string; description?: string; } +/** + * One of a provider section's several url/key groups. `label` is the idempotency key the write methods address an entry by. + * + * This interface was referenced by `RavenRpcRoot`'s JSON-Schema + * via the `definition` "ProviderEndpointInfo". + */ +export interface ProviderEndpointInfo { + label: string; + /** + * Redacted for display: `****set****` or `(empty)`. + */ + api_key: string; + api_base?: string; + extra_headers?: { + [k: string]: string; + }; +} /** * This interface was referenced by `RavenRpcRoot`'s JSON-Schema * via the `definition` "UsageSnapshot". @@ -866,6 +883,55 @@ export interface ModelRemoveModelParams { export interface ModelRemoveModelResult { provider: ModelOptionProvider; } +/** + * This interface was referenced by `RavenRpcRoot`'s JSON-Schema + * via the `definition` "ModelEndpointsParams". + */ +export interface ModelEndpointsParams { + slug: string; + session_id?: string; +} +/** + * This interface was referenced by `RavenRpcRoot`'s JSON-Schema + * via the `definition` "ModelEndpointsResult". + */ +export interface ModelEndpointsResult { + endpoints: ProviderEndpointInfo[]; +} +/** + * This interface was referenced by `RavenRpcRoot`'s JSON-Schema + * via the `definition` "ModelAddEndpointParams". + */ +export interface ModelAddEndpointParams { + slug: string; + label: string; + api_key?: string; + api_base?: string; + session_id?: string; +} +/** + * This interface was referenced by `RavenRpcRoot`'s JSON-Schema + * via the `definition` "ModelAddEndpointResult". + */ +export interface ModelAddEndpointResult { + endpoints: ProviderEndpointInfo[]; +} +/** + * This interface was referenced by `RavenRpcRoot`'s JSON-Schema + * via the `definition` "ModelRemoveEndpointParams". + */ +export interface ModelRemoveEndpointParams { + slug: string; + label: string; + session_id?: string; +} +/** + * This interface was referenced by `RavenRpcRoot`'s JSON-Schema + * via the `definition` "ModelRemoveEndpointResult". + */ +export interface ModelRemoveEndpointResult { + endpoints: ProviderEndpointInfo[]; +} /** * This interface was referenced by `RavenRpcRoot`'s JSON-Schema * via the `definition` "ConfigGetParams". diff --git a/ui-tui/src/types.ts b/ui-tui/src/types.ts index 3d0de63b..ca8b29f7 100644 --- a/ui-tui/src/types.ts +++ b/ui-tui/src/types.ts @@ -190,6 +190,9 @@ export interface McpServerStatus { export interface SessionInfo { context_window?: number cwd?: string + // Which of a multi-endpoint provider's endpoints this session is on; + // absent for single-endpoint providers, which have no label worth showing. + endpoint?: string | null fast?: boolean lazy?: boolean mcp_servers?: McpServerStatus[] From b59f492bea9b6585ba7f96ed4e9f38086a615f9a Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 01:26:57 +0800 Subject: [PATCH 29/78] fix(providers): rotate endpoints on auth failures too Caught on a live wire, not in the mocks: a dead OpenRouter key classifies auth, and auth deliberately never falls back across models -- the same key fails for every model -- so the rotor, judging hops by should_fallback, returned the 401 with a healthy endpoint sitting right behind it. A revoked or exhausted key on one account says nothing about the next account, and that failover is the reason a second endpoint exists. The rotor now judges continuation by its own predicate: should_fallback or auth rotates, while genuinely endpoint-agnostic failures (invalid_request, context overflow, unknown) still return immediately. Co-authored-by: Claude (claude-fable-5) --- raven/providers/endpoint_rotor.py | 31 ++++++++++++++------ tests/test_provider_endpoint_rotor.py | 41 +++++++++++++++++++++++---- 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/raven/providers/endpoint_rotor.py b/raven/providers/endpoint_rotor.py index fb1fae68..2cbda755 100644 --- a/raven/providers/endpoint_rotor.py +++ b/raven/providers/endpoint_rotor.py @@ -29,7 +29,7 @@ from dataclasses import dataclass, field from typing import Any -from raven.providers.base import LLMProvider, LLMResponse, StreamDelta +from raven.providers.base import ErrorClassification, LLMProvider, LLMResponse, StreamDelta from raven.providers.endpoints import ResolvedEndpoint #: Seconds a failed endpoint sits out before it is tried again, doubling per @@ -41,6 +41,21 @@ _COOLDOWN_CAP_SECONDS = 300.0 +def _rotates(classification: ErrorClassification) -> bool: + """Whether this failure is worth trying the next endpoint for. + + Broader than ``should_fallback`` in exactly one place: an ``auth`` failure + never falls back across *models* -- the same key fails for every model -- + but each endpoint here is its own account with its own key, and a revoked + or exhausted key on one account says nothing about the next. Measured on + a live wire: a dead OpenRouter key classifies ``auth``, and judging it by + ``should_fallback`` alone left the rotor returning the 401 with a healthy + endpoint sitting right behind it. Everything genuinely endpoint-agnostic + (invalid_request, context overflow, unknown) still returns immediately. + """ + return classification.should_fallback or classification.category == "auth" + + @dataclass class RotorState: """Per-instance rotation/failover bookkeeping -- process memory only. @@ -183,10 +198,10 @@ async def _chat_attempt_with_retry( next attempt. A fallback-worthy exhaustion moves to the next endpoint from ``_healthy_order()``; a non-fallback error (auth, invalid_request, context overflow, ...) returns immediately, since a - different endpoint on the same account/vendor will not fix a rejected - key or a malformed request. Exhausting every endpoint returns the - last response, letting the caller's own model-chain fallback - (``LLMProvider.chat_with_retry``) take over from there. + different endpoint on the same vendor will not fix a malformed + request. Exhausting every endpoint returns the last response, letting + the caller's own model-chain fallback (``LLMProvider.chat_with_retry``) + take over from there. """ order = self._healthy_order() last_response: LLMResponse | None = None @@ -207,7 +222,7 @@ async def _chat_attempt_with_retry( classification = response.error_classification or self.classify_error(content=response.content) response.error_classification = classification last_response = response - if not classification.should_fallback: + if not _rotates(classification): return response self._mark_failure(i) @@ -257,7 +272,7 @@ async def chat_stream( return except Exception as exc: classification = self.classify_error(exc) - if not classification.should_fallback: + if not _rotates(classification): raise self._mark_failure(i) last_failure = exc @@ -268,7 +283,7 @@ async def chat_stream( # a terminal error delta rather than an exception; judged # the same way as one. classification = first.error_classification or self.classify_error(content=first.content) - if classification.should_fallback: + if _rotates(classification): self._mark_failure(i) last_failure = first continue diff --git a/tests/test_provider_endpoint_rotor.py b/tests/test_provider_endpoint_rotor.py index 47046421..04722e9f 100644 --- a/tests/test_provider_endpoint_rotor.py +++ b/tests/test_provider_endpoint_rotor.py @@ -4,7 +4,7 @@ - sticky: first endpoint stays "the" endpoint until it fails, cooldown transfers to the next, and expiry restores it - round_robin: the cursor advances by one on every call -- a non-fallback error (auth) never rotates +- an endpoint-agnostic error (invalid_request) never rotates; auth does - every endpoint cooling still dispatches (no deadlock) - chat_stream: transfer before the first delta, no transfer after it, no token replay @@ -178,11 +178,15 @@ async def test_active_endpoint_label_skips_a_cooling_endpoint(clock): assert rotor.active_endpoint_label == "e0" -async def test_non_fallback_error_returns_immediately_without_rotating(clock): +async def test_endpoint_agnostic_error_returns_immediately_without_rotating(clock): e0 = _StubInner( "e0", chat_script=[ - LLMResponse(content="401 unauthorized", finish_reason="error", error_classification=_NON_FALLBACK_FATAL) + LLMResponse( + content="400 invalid request", + finish_reason="error", + error_classification=ErrorClassification(category="invalid_request"), + ) ], ) e1 = _StubInner("e1") @@ -191,13 +195,38 @@ async def test_non_fallback_error_returns_immediately_without_rotating(clock): resp = await rotor.chat_with_retry(messages=[], model="m") assert resp.finish_reason == "error" - assert resp.content == "401 unauthorized" + assert resp.content == "400 invalid request" assert (e0.chat_calls, e1.chat_calls) == (1, 0) - # A fatal-but-not-fallback-worthy error must not cool the endpoint either -- - # there was nothing wrong with the endpoint, the request was invalid. + # A fatal-but-endpoint-agnostic error must not cool the endpoint either -- + # there was nothing wrong with the endpoint, the request was malformed. assert rotor._state.is_cooling(0, clock.now) is False +async def test_an_auth_failure_rotates_to_the_next_account(clock): + """A dead key is exactly the failure a second account exists for. + + auth never falls back across models -- the same key fails for every + model -- but each endpoint is its own account with its own key, so the + rotor judges it by its own predicate. Measured on a live OpenRouter + wire before this test existed: judging by should_fallback alone + returned the 401 with a healthy endpoint sitting right behind it. + """ + e0 = _StubInner( + "e0", + chat_script=[ + LLMResponse(content="401 unauthorized", finish_reason="error", error_classification=_NON_FALLBACK_FATAL) + ], + ) + e1 = _StubInner("e1") + rotor = _make_rotor([e0, e1], strategy="sticky") + + resp = await rotor.chat_with_retry(messages=[], model="m") + + assert resp.finish_reason != "error" + assert (e0.chat_calls, e1.chat_calls) == (1, 1) + assert rotor._state.is_cooling(0, clock.now) is True + + async def test_all_endpoints_cooling_still_dispatches_in_order(clock): e0 = _StubInner("e0", chat_script=[LLMResponse(content="ok0", finish_reason="stop")]) e1 = _StubInner("e1") From 42b7945f4e4e1208329919c0fede1b0b636bebe2 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 01:33:55 +0800 Subject: [PATCH 30/78] fix(*): survive duck-typed providers and argue the split into the wire guard The full-suite sweep caught two faces no executor's test selection covered. The stream collation called emits_unparsed_reasoning directly, but the loop accepts duck-typed providers -- fourteen tests' stubs implement just chat/chat_stream -- so the call now reads the attribute and treats absence as the LLMProvider default. And the wire-form invariant flags the everos module the wizard split created, for the same prefix-stripping read onboard_commands was already argued in for. Co-authored-by: Claude (claude-fable-5) --- raven/agent/loop/main.py | 6 +++++- tests/test_provider_resolution_invariants.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/raven/agent/loop/main.py b/raven/agent/loop/main.py index ff5a5d26..ae6e4ac9 100644 --- a/raven/agent/loop/main.py +++ b/raven/agent/loop/main.py @@ -1632,7 +1632,11 @@ async def _llm_call_stream( content = "".join(content_buf) reasoning_content = "".join(reasoning_buf) or None - if reasoning_content is None and self.provider.emits_unparsed_reasoning(): + # getattr because the loop accepts duck-typed providers (test stubs and + # thin adapters implement just chat/chat_stream); absent means the + # LLMProvider default, False. + emits_unparsed = getattr(self.provider, "emits_unparsed_reasoning", None) + if reasoning_content is None and emits_unparsed is not None and emits_unparsed(): split_reasoning, content = split_orphan_think(content) reasoning_content = split_reasoning diff --git a/tests/test_provider_resolution_invariants.py b/tests/test_provider_resolution_invariants.py index 54e4c06b..e5bb81ae 100644 --- a/tests/test_provider_resolution_invariants.py +++ b/tests/test_provider_resolution_invariants.py @@ -317,6 +317,9 @@ def test_the_wire_form_of_a_model_id_is_built_in_one_module() -> None: # Strips a known prefix off a model id to recover the vendor's own id for # a connectivity probe. Also decomposition. "raven/cli/onboard_commands.py", + # Carries the wizard's EverOS cluster split out of onboard_commands -- + # the same prefix-stripping read, same argument, new file name. + "raven/cli/onboard_everos.py", } needles = (".model_prefix", ".skip_prefixes", '"model_prefix"', '"skip_prefixes"') offenders = { From 70e2624cd4acbea2114449ac7b368eb8420ed7aa Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 01:40:19 +0800 Subject: [PATCH 31/78] test(providers): count the rotor as the sixth concrete backend The catalogue guard enumerates concrete LLMProvider subclasses and passed standalone only because nothing had imported endpoint_rotor yet -- the full suite imports it and the set gained a member. The rotor is a real backend in dispatch terms (make_provider returns it for a multi-endpoint section), so it joins the expected set, imported explicitly like its five siblings to keep the walk deterministic. Co-authored-by: Claude (claude-fable-5) --- tests/test_provider_catalog.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_provider_catalog.py b/tests/test_provider_catalog.py index 71fc8582..6f9a0377 100644 --- a/tests/test_provider_catalog.py +++ b/tests/test_provider_catalog.py @@ -137,6 +137,7 @@ def _concrete_provider_subclasses() -> set[type]: import raven.providers.azure_openai_provider # noqa: F401 import raven.providers.litellm_provider # noqa: F401 import raven.providers.minimax_oauth_provider # noqa: F401 + import raven.providers.endpoint_rotor # noqa: F401 import raven.providers.openai_codex_provider # noqa: F401 import raven.providers.per_model_provider # noqa: F401 @@ -157,9 +158,10 @@ def _concrete_provider_subclasses() -> set[type]: return seen -def test_exactly_five_concrete_backend_classes() -> None: +def test_exactly_six_concrete_backend_classes() -> None: # This asserts class existence only, not the dispatch wiring. from raven.providers.azure_openai_provider import AzureOpenAIProvider + from raven.providers.endpoint_rotor import EndpointRotorProvider from raven.providers.litellm_provider import LiteLLMProvider from raven.providers.minimax_oauth_provider import MiniMaxOAuthProvider from raven.providers.openai_codex_provider import OpenAICodexProvider @@ -171,6 +173,10 @@ def test_exactly_five_concrete_backend_classes() -> None: OpenAICodexProvider, MiniMaxOAuthProvider, PerModelProvider, + # Multi-endpoint rotation/failover wrapper (#143/#144): a real backend + # in dispatch terms -- make_provider returns it for a section that + # resolves to more than one endpoint. + EndpointRotorProvider, } assert _concrete_provider_subclasses() == expected for cls in expected: From ede6bf12e185916ed864409679f67630c8ed27a1 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 01:42:01 +0800 Subject: [PATCH 32/78] chore(providers): drop issue references from source comments Two comments this branch added named issue numbers, which the repo's comment rules keep out of source; both now describe the mechanism instead. Co-authored-by: Claude (claude-fable-5) --- raven/providers/endpoint_rotor.py | 4 ++-- tests/test_provider_catalog.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/raven/providers/endpoint_rotor.py b/raven/providers/endpoint_rotor.py index 2cbda755..a81afb24 100644 --- a/raven/providers/endpoint_rotor.py +++ b/raven/providers/endpoint_rotor.py @@ -2,8 +2,8 @@ ``provider_endpoints`` (see ``raven.providers.endpoints``) resolves a section into one or more ``ResolvedEndpoint``s -- several accounts on the same -vendor, several regions, several keys. This module is what #143/#144 asked -for on top of that list: spread requests across them (round-robin) or stick +vendor, several regions, several keys. This module is the behavior asked +of that list: spread requests across them (round-robin) or stick to one until it misbehaves (sticky), and route around an endpoint that just failed instead of sending the next request into the same wall. diff --git a/tests/test_provider_catalog.py b/tests/test_provider_catalog.py index 6f9a0377..fee11d27 100644 --- a/tests/test_provider_catalog.py +++ b/tests/test_provider_catalog.py @@ -135,9 +135,9 @@ def _concrete_provider_subclasses() -> set[type]: """All non-abstract LLMProvider subclasses defined in raven.providers.""" # Import each backend module so its subclass is registered on LLMProvider. import raven.providers.azure_openai_provider # noqa: F401 + import raven.providers.endpoint_rotor # noqa: F401 import raven.providers.litellm_provider # noqa: F401 import raven.providers.minimax_oauth_provider # noqa: F401 - import raven.providers.endpoint_rotor # noqa: F401 import raven.providers.openai_codex_provider # noqa: F401 import raven.providers.per_model_provider # noqa: F401 @@ -173,9 +173,9 @@ def test_exactly_six_concrete_backend_classes() -> None: OpenAICodexProvider, MiniMaxOAuthProvider, PerModelProvider, - # Multi-endpoint rotation/failover wrapper (#143/#144): a real backend - # in dispatch terms -- make_provider returns it for a section that - # resolves to more than one endpoint. + # Multi-endpoint rotation/failover wrapper: a real backend in dispatch + # terms -- make_provider returns it for a section that resolves to + # more than one endpoint. EndpointRotorProvider, } assert _concrete_provider_subclasses() == expected From 69f6a54eaac60c77ff0c7bfcaadc51cfe073ec6d Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 01:59:34 +0800 Subject: [PATCH 33/78] fix(providers): stop reading an unresolved identity as self-hosted emits_unparsed_reasoning treated "provider_name resolves to no spec" as the self-hosted shape and normalized think tags there, while can_serve -- fixed in the same batch -- settled the opposite reading for the same condition: an unresolved name says nothing, and real constructors (the proactive planner, the evolver) build direct big-vendor connections with no provider_name at all, so guessing self-hosted re-opened the ordinary-content cut this gate exists to close. Both predicates now read unresolved identity the same way: do not guess. Co-authored-by: Claude (claude-fable-5) --- raven/providers/litellm_provider.py | 21 +++++++++++++-------- tests/test_litellm_provider_attribution.py | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/raven/providers/litellm_provider.py b/raven/providers/litellm_provider.py index dd3658c5..b8ac87b8 100644 --- a/raven/providers/litellm_provider.py +++ b/raven/providers/litellm_provider.py @@ -229,16 +229,21 @@ def emits_unparsed_reasoning(self) -> bool: is just content; the generic ``custom`` endpoint and a local spec (hosted_vllm, ollama_chat) *are* the self-hosted inference server this normalization exists for. When nothing was auto-detected, fall back to - whatever spec ``provider_name`` resolves to -- no spec at all (a bare - passthrough LiteLLM has no entry for) is the same self-hosted shape as - an explicit ``custom`` endpoint. Only a resolved spec that is neither - local nor ``custom`` -- a known direct big-vendor connection - (anthropic, openai, deepseek, ...) -- answers False: the - parser-less sglang/vLLM shape only comes from a self-hosted backend, - never from a vendor serving its own model behind its own API. + whatever spec ``provider_name`` resolves to. + + An identity that resolves to nothing answers False, the same reading + ``can_serve`` settled on: an unresolved name says nothing about the + backend, and several production constructors (the proactive planner, + the evolver) build direct big-vendor connections with no + ``provider_name`` at all -- guessing "self-hosted" there re-opens the + false-positive cut on ordinary content this gate exists to close. A + genuinely self-hosted backend is reached through ``custom`` or a + local spec, which is where the parser-less sglang/vLLM shape comes + from; a resolved direct big vendor (anthropic, openai, ...) never + produces it behind its own API. """ spec = self._gateway or find_by_name(canonical_provider_name(self._provider_name)) - return spec is None or spec.is_local or spec.name == "custom" + return spec is not None and (spec.is_local or spec.name == "custom") def _supports_cache_control(self, model: str) -> bool: """Return True when this request may carry cache_control blocks. diff --git a/tests/test_litellm_provider_attribution.py b/tests/test_litellm_provider_attribution.py index b255e678..c11c41d3 100644 --- a/tests/test_litellm_provider_attribution.py +++ b/tests/test_litellm_provider_attribution.py @@ -111,6 +111,24 @@ def test_parse_response_leaves_structured_reasoning_content_alone(): assert result.content == "visible\nanswer" +def test_parse_response_leaves_bare_close_tag_alone_for_an_unresolved_identity(): + """An identity that resolves to no spec says nothing about the backend. + + The proactive planner and the evolver both build direct big-vendor + connections with no provider_name at all; reading "no spec" as + "self-hosted" re-opened the ordinary-content cut on exactly those + constructors, so the gate answers False there -- same reading as + can_serve's. + """ + provider = _make_provider("fireworks") + response = _fake_response("discussing the tag in my answer") + + result = provider._parse_response(response) + + assert result.reasoning_content is None + assert result.content == "discussing the tag in my answer" + + def test_parse_response_leaves_bare_close_tag_alone_for_direct_anthropic(): provider = _make_provider("anthropic") response = _fake_response("discussing the tag in my answer") From 28ed608cf9bd3af4a3d4430bc604025edac5b376 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 02:00:13 +0800 Subject: [PATCH 34/78] docs(context): register the provider endpoint term "endpoint" now has three same-sounding neighbors in the repo -- a Provider Endpoint (an account's url/key/headers group under one provider), routing's ModelEndpoint (a backend picked per model), and a bare api_base (one endpoint's address). The new concept gets its glossary entry with the disambiguation spelled out, as the domain-term rule requires for a coined term. Co-authored-by: Claude (claude-fable-5) --- CONTEXT.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CONTEXT.md b/CONTEXT.md index 4afc98e5..32bedf0d 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -287,6 +287,17 @@ vendor with the old vendor's key. _Avoid_: reading it as a provider *signal* -- a pinned name says which section to ask about, never that the section holds credentials. +**Provider Endpoint**: +One url/key/headers group a provider section offers, of possibly several +(`ProviderConfig.endpoints`, resolved through `providers/endpoints.py::provider_endpoints` +whichever spelling the section used -- explicit list, Gemini's `api_key_list`, or the +flat fields). Several endpoints on one section mean several accounts on the same vendor; +`EndpointRotorProvider` spreads and fails over across them. +_Avoid_: two same-sounding neighbors. Routing's `ModelEndpoint` (`RoutingConfig.models`) +keys by *model* and picks a backend per request; a Provider Endpoint keys by *account* +under one provider. And a bare `api_base` is one endpoint's address, not the endpoint -- +an endpoint is the whole credential group under a label. + ### TUI-RPC **TUI-RPC**: From 2f7c3af991e94eb9e457c36e6158223af569c70a Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 02:06:35 +0800 Subject: [PATCH 35/78] fix(*): carry endpoint material and hooks through the assembly layer Review round two probed the assembly seams the unit layer never exercises and found four holes. A section with exactly one explicit endpoint silently lost its key and base -- the single-endpoint branch still read the flat fields, the declared-but-dead shape again -- so it now draws from the resolved endpoint, byte-equivalent for flat configs. LazyProvider, which raven tui wraps every provider in, forwarded only the four call methods, killing orphan-think normalization and the endpoint footer on the main path; it now forwards emits_unparsed_reasoning post-build and answers the endpoint label from the wiring-time initial before the first call builds the inner. The rotor propagates generation settings to every inner the way PerModelProvider already did, so a configured timeout survives, and delegates emits_unparsed_reasoning to its inners for the same reason as can_serve. Co-authored-by: Claude (claude-fable-5) --- raven/cli/_helpers.py | 18 ++++ raven/providers/endpoint_rotor.py | 33 ++++++- raven/providers/lazy.py | 23 +++++ tests/test_cli_helpers.py | 122 ++++++++++++++++++++++++++ tests/test_lazy_provider.py | 70 ++++++++++++++- tests/test_provider_endpoint_rotor.py | 34 +++++++ 6 files changed, 298 insertions(+), 2 deletions(-) diff --git a/raven/cli/_helpers.py b/raven/cli/_helpers.py index 8d9b4e3a..634f5bc0 100644 --- a/raven/cli/_helpers.py +++ b/raven/cli/_helpers.py @@ -177,6 +177,17 @@ def make_inner(ep): default_model=model, strategy=p.endpoint_strategy if p else "sticky", ) + elif eps: + extra_body = wire_overrides(provider_name, model) or None + provider = LiteLLMProvider( + api_key=eps[0].api_key, + api_base=eps[0].api_base or config.get_api_base(model), + default_model=model, + extra_headers=eps[0].extra_headers or (p.extra_headers if p else None), + provider_name=provider_name, + extra_body=extra_body, + model_overrides=config.agents.defaults.model_overrides, + ) else: extra_body = wire_overrides(provider_name, model) or None provider = LiteLLMProvider( @@ -204,10 +215,16 @@ def make_lazy_provider(config: Config): call, so AgentLoop construction stays fast. Credentials are checked now (fail-fast preserved) and the real provider is pre-warmed in the background.""" from raven.providers.base import GenerationSettings + from raven.providers.endpoints import provider_endpoints from raven.providers.lazy import LazyProvider check_provider_credentials(config) defaults = config.agents.defaults + + p = config.get_provider(defaults.model) + eps = provider_endpoints(p) if p else [] + initial_endpoint_label = eps[0].label if len(eps) > 1 else None + provider = LazyProvider( factory=lambda: make_provider(config), default_model=defaults.model, @@ -217,6 +234,7 @@ def make_lazy_provider(config: Config): reasoning_effort=defaults.reasoning_effort, timeout=defaults.llm_call_timeout, ), + initial_endpoint_label=initial_endpoint_label, ) provider.prewarm() return provider diff --git a/raven/providers/endpoint_rotor.py b/raven/providers/endpoint_rotor.py index a81afb24..b4695858 100644 --- a/raven/providers/endpoint_rotor.py +++ b/raven/providers/endpoint_rotor.py @@ -29,7 +29,7 @@ from dataclasses import dataclass, field from typing import Any -from raven.providers.base import ErrorClassification, LLMProvider, LLMResponse, StreamDelta +from raven.providers.base import ErrorClassification, GenerationSettings, LLMProvider, LLMResponse, StreamDelta from raven.providers.endpoints import ResolvedEndpoint #: Seconds a failed endpoint sits out before it is tried again, doubling per @@ -127,6 +127,31 @@ def __init__( self._default_model = default_model self.strategy = strategy self._state = RotorState() + self.generation = self.generation # push the base class's default down now that inners exist + + @property + def generation(self) -> GenerationSettings: + return self._generation + + @generation.setter + def generation(self, value: GenerationSettings) -> None: + """Push generation settings down to every inner. + + ``make_provider`` builds this instance and only then assigns + ``provider.generation = GenerationSettings(...)`` from config (see + ``per_model_provider.py``'s ``PerModelProvider.__init__`` for the same + push-down at construction time) -- without this setter that assignment + would land on the rotor alone and every inner would keep answering + temperature/max_tokens/timeout from its own untouched default. + + ``getattr(self, "_inners", [])`` covers the one call that happens + before ``self._inners`` exists: the base class's own ``__init__`` + assigns a default ``self.generation`` before this subclass's + constructor has built the endpoint list. + """ + self._generation = value + for inner in getattr(self, "_inners", []): + inner.generation = value def _mark_failure(self, i: int) -> None: self._state.mark_failure(i, time.monotonic()) @@ -307,5 +332,11 @@ def can_serve(self, model: str) -> bool: purposes is one answer, not one per endpoint.""" return self._inners[0].can_serve(model) + def emits_unparsed_reasoning(self) -> bool: + """Delegates to the first endpoint's inner, same reasoning as ``can_serve``: + every endpoint under one rotor is the same vendor/section, so the shape + of its wire is one answer, not one per endpoint.""" + return self._inners[0].emits_unparsed_reasoning() + def get_default_model(self) -> str: return self._default_model diff --git a/raven/providers/lazy.py b/raven/providers/lazy.py index bce0ba55..34c17e8c 100644 --- a/raven/providers/lazy.py +++ b/raven/providers/lazy.py @@ -24,11 +24,14 @@ def __init__( factory: Callable[[], LLMProvider], default_model: str, generation: GenerationSettings, + *, + initial_endpoint_label: str | None = None, ): super().__init__() self._factory = factory self._default_model = default_model self.generation = generation + self._initial_endpoint_label = initial_endpoint_label self._provider: LLMProvider | None = None self._lock = threading.Lock() @@ -56,6 +59,26 @@ def _run() -> None: def get_default_model(self) -> str: return self._default_model + def emits_unparsed_reasoning(self) -> bool: + """Forwarded post-materialization: the stream collation that asks this + only runs after a call, and the first call is what builds the inner + provider -- before that there is nothing to normalize anyway.""" + return False if self._provider is None else self._provider.emits_unparsed_reasoning() + + @property + def active_endpoint_label(self) -> str | None: + """Which endpoint is answering, for the session footer. + + Before materialization, the rotor behind the real provider has not + rotated yet, so the first endpoint it would pick (``initial_endpoint_label``) + is exactly what a sticky rotor would answer -- no build needed just to + display a label. Once built, defer to the inner provider so the footer + reflects any rotation that happened since. + """ + if self._provider is None: + return self._initial_endpoint_label + return getattr(self._provider, "active_endpoint_label", None) + async def chat(self, *args: Any, **kwargs: Any) -> LLMResponse: return await self._built().chat(*args, **kwargs) diff --git a/tests/test_cli_helpers.py b/tests/test_cli_helpers.py index d2a5ac25..96efc34a 100644 --- a/tests/test_cli_helpers.py +++ b/tests/test_cli_helpers.py @@ -158,6 +158,58 @@ def test_make_lazy_provider_returns_lazy_without_building(monkeypatch: pytest.Mo assert provider.get_default_model() == "my-model" +def test_make_lazy_provider_carries_the_first_endpoint_label_for_a_multi_endpoint_section( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Session footer reads ``active_endpoint_label`` before the first call, when + the real (rotor-wrapping) provider has not been built yet. For a section + with several endpoints, ``make_lazy_provider`` must hand the lazy wrapper + the first entry's label so that read answers something instead of + always ``None``.""" + from raven.config.schema import Config + from raven.providers.lazy import LazyProvider + + monkeypatch.setattr(_helpers, "make_provider", lambda _c: SimpleNamespace(name="real")) + # Real prewarm races a background thread against this assertion -- disable + # it so the test observes the pre-materialization state deterministically. + monkeypatch.setattr(LazyProvider, "prewarm", lambda self: None) + config = Config.model_validate( + { + "providers": { + "custom": { + "endpoints": [ + {"label": "first", "apiKey": "k1", "apiBase": "https://first.example"}, + {"label": "second", "apiKey": "k2", "apiBase": "https://second.example"}, + ] + } + }, + "agents": {"defaults": {"model": "my-model", "provider": "custom"}}, + } + ) + + provider = _helpers.make_lazy_provider(config) + + assert isinstance(provider, LazyProvider) + assert provider.active_endpoint_label == "first" + + +def test_make_lazy_provider_has_no_endpoint_label_for_a_single_endpoint_section( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A section with only one endpoint (flat or explicit) never rotates, so + there is nothing for the footer to name -- ``active_endpoint_label`` is + ``None`` rather than a label nobody will ever see it change from.""" + from raven.providers.lazy import LazyProvider + + monkeypatch.setattr(_helpers, "make_provider", lambda _c: SimpleNamespace(name="real")) + from raven.config.loader import load_config + + provider = _helpers.make_lazy_provider(load_config(_write_config(tmp_path, api_key="sk-x"))) + + assert isinstance(provider, LazyProvider) + assert provider.active_endpoint_label is None + + @pytest.mark.parametrize( ("provider", "model", "expected"), [ @@ -309,6 +361,76 @@ def test_make_provider_a_single_endpoint_entry_still_returns_a_plain_provider( assert type(provider) is LiteLLMProvider +def test_make_provider_single_endpoint_entry_credentials_are_not_dropped( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A section declaring exactly one entry under ``endpoints`` has nothing in + the flat ``apiKey``/``apiBase``/``extraHeaders`` fields -- reading those + instead of the endpoint (the shape this section actually used) silently + sent an empty key. The single-endpoint path must read the endpoint.""" + from raven.config.schema import Config + from raven.providers.litellm_provider import LiteLLMProvider + + config = Config.model_validate( + { + "providers": { + "custom": { + "endpoints": [ + { + "label": "only", + "apiKey": "k-only", + "apiBase": "https://only.example", + "extraHeaders": {"X-Only": "1"}, + } + ] + } + }, + "agents": {"defaults": {"model": "my-model", "provider": "custom"}}, + } + ) + monkeypatch.setattr("raven.cli._helpers.check_provider_credentials", lambda _config: None) + + provider = _helpers.make_provider(config) + + assert type(provider) is LiteLLMProvider + # The reverse-case shape a prior review caught live: this must not be empty. + assert provider.api_key == "k-only" + assert provider.api_base == "https://only.example" + assert provider.extra_headers == {"X-Only": "1"} + + +def test_make_provider_flat_config_is_equivalent_through_the_endpoint_path( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A plain flat ``apiKey``/``apiBase`` section (no ``endpoints`` field) is + synthesized by ``provider_endpoints`` into a single endpoint and now goes + through the same single-endpoint-material path as an explicit one. The + result must match what the flat fields alone produced before this change.""" + from raven.config.schema import Config + from raven.providers.litellm_provider import LiteLLMProvider + + config = Config.model_validate( + { + "providers": { + "custom": { + "apiKey": "k-flat", + "apiBase": "https://flat.example", + "extraHeaders": {"X-Flat": "1"}, + } + }, + "agents": {"defaults": {"model": "my-model", "provider": "custom"}}, + } + ) + monkeypatch.setattr("raven.cli._helpers.check_provider_credentials", lambda _config: None) + + provider = _helpers.make_provider(config) + + assert type(provider) is LiteLLMProvider + assert provider.api_key == "k-flat" + assert provider.api_base == "https://flat.example" + assert provider.extra_headers == {"X-Flat": "1"} + + @pytest.mark.parametrize( ("provider", "model", "extra_section"), [ diff --git a/tests/test_lazy_provider.py b/tests/test_lazy_provider.py index 2e55d0ee..6656e538 100644 --- a/tests/test_lazy_provider.py +++ b/tests/test_lazy_provider.py @@ -11,12 +11,18 @@ class _FakeProvider: - def __init__(self) -> None: + def __init__(self, *, emits_unparsed_reasoning: bool = False, active_endpoint_label: str | None = None) -> None: self.generation = GenerationSettings() + self._emits_unparsed_reasoning = emits_unparsed_reasoning + if active_endpoint_label is not None: + self.active_endpoint_label = active_endpoint_label def get_default_model(self) -> str: return "built-model" + def emits_unparsed_reasoning(self) -> bool: + return self._emits_unparsed_reasoning + async def chat(self, *args, **kwargs) -> str: return "chat" @@ -98,6 +104,68 @@ def factory() -> _FakeProvider: assert built.wait(timeout=2.0), "prewarm did not build the provider in the background" +def test_emits_unparsed_reasoning_defaults_false_before_materialization() -> None: + """The stream collation that asks this only runs after a call, and the + first call is what builds the inner provider -- before that there is + nothing to normalize anyway, so the answer must be False without ever + invoking the factory.""" + calls: list = [] + + def factory() -> _FakeProvider: + calls.append(1) + return _FakeProvider(emits_unparsed_reasoning=True) + + lp = LazyProvider(factory, "cfg-model", GenerationSettings()) + + assert lp.emits_unparsed_reasoning() is False + assert calls == [] # asking did not build the provider + + +def test_emits_unparsed_reasoning_forwards_after_materialization() -> None: + """Once built, the answer is the real provider's -- not the pre-build + default -- so the TUI's think-tag normalization (which reads this) keeps + working on the primary path through ``make_lazy_provider``.""" + + def factory() -> _FakeProvider: + return _FakeProvider(emits_unparsed_reasoning=True) + + lp = LazyProvider(factory, "cfg-model", GenerationSettings()) + asyncio.run(lp.chat([])) # materializes _provider + + assert lp.emits_unparsed_reasoning() is True + + +def test_active_endpoint_label_is_the_initial_label_before_materialization() -> None: + lp = LazyProvider( + lambda: _FakeProvider(), + "cfg-model", + GenerationSettings(), + initial_endpoint_label="first", + ) + + assert lp.active_endpoint_label == "first" + + +def test_active_endpoint_label_is_none_when_no_initial_label_was_given() -> None: + lp = LazyProvider(lambda: _FakeProvider(), "cfg-model", GenerationSettings()) + + assert lp.active_endpoint_label is None + + +def test_active_endpoint_label_forwards_to_the_real_provider_after_materialization() -> None: + """After the first call, the rotor behind the real provider may have + rotated -- the footer must reflect that, not stay pinned to the initial + label forever.""" + + def factory() -> _FakeProvider: + return _FakeProvider(active_endpoint_label="second") + + lp = LazyProvider(factory, "cfg-model", GenerationSettings(), initial_endpoint_label="first") + asyncio.run(lp.chat([])) + + assert lp.active_endpoint_label == "second" + + def test_prewarm_swallows_build_error() -> None: def factory(): raise RuntimeError("boom") diff --git a/tests/test_provider_endpoint_rotor.py b/tests/test_provider_endpoint_rotor.py index 04722e9f..1878b244 100644 --- a/tests/test_provider_endpoint_rotor.py +++ b/tests/test_provider_endpoint_rotor.py @@ -309,6 +309,40 @@ async def test_chat_attempt_with_retry_composes_with_base_model_chain_fallback(c assert (e0.chat_calls, e1.chat_calls) == (2, 1) +async def test_generation_assigned_after_construction_propagates_to_every_inner(clock): + """``make_provider`` builds the rotor, then assigns ``provider.generation = + GenerationSettings(...)`` from config -- see ``raven/cli/_helpers.py``. + Without push-down each inner keeps the base class's untouched default + (600s timeout, temperature 0.7, ...), so a configured timeout is silently + ignored on every actual request.""" + from raven.providers.base import GenerationSettings + + e0 = _StubInner("e0") + e1 = _StubInner("e1") + rotor = _make_rotor([e0, e1], strategy="sticky") + + settings = GenerationSettings(temperature=0.1, max_tokens=99, timeout=12.5) + rotor.generation = settings + + assert e0.generation is settings + assert e1.generation is settings + assert e0.generation.timeout == 12.5 + assert e1.generation.timeout == 12.5 + + +def test_emits_unparsed_reasoning_delegates_to_the_first_inner(clock): + """Same reasoning as ``can_serve``: every endpoint under one rotor is the + same vendor/section, so the shape of its wire is one answer, not one per + endpoint.""" + e0 = _StubInner("e0") + e0.emits_unparsed_reasoning = lambda: True + e1 = _StubInner("e1") + e1.emits_unparsed_reasoning = lambda: False + rotor = _make_rotor([e0, e1], strategy="sticky") + + assert rotor.emits_unparsed_reasoning() is True + + async def test_cooldown_doubles_per_failure_capped_and_clears_on_success(clock): state = RotorState() From f24bc26d85e3435c73e3bf6de388b561f0684395 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 02:09:21 +0800 Subject: [PATCH 36/78] fix(*): redact endpoint keys and align the display and validation faces The redaction walk keyed off flat field names, so raven provider get printed every endpoint api_key in plaintext on its redacting default -- each list element now passes through the same secret-field rule. provider list showed a checkmark next to "(empty)" for an endpoints-only section; it now says the keys live in the endpoints. Duplicate endpoint labels are rejected by a schema validator instead of a docstring promise. And the cache-only tier of the OpenRouter lookup no longer calls through the rebindable fetch name with a keyword a zero-argument test double cannot accept. Co-authored-by: Claude (claude-fable-5) --- raven/config/schema.py | 14 ++++++- raven/config/update_providers.py | 25 +++++++++++- raven/providers/rates.py | 56 ++++++++++++++++----------- tests/test_config_schema.py | 14 +++++++ tests/test_config_update_providers.py | 41 ++++++++++++++++++++ tests/test_provider_rates.py | 21 ++++++++++ 6 files changed, 146 insertions(+), 25 deletions(-) diff --git a/raven/config/schema.py b/raven/config/schema.py index 65298242..778de815 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Any, Literal -from pydantic import AliasChoices, BaseModel, ConfigDict, Field, model_validator +from pydantic import AliasChoices, BaseModel, ConfigDict, Field, field_validator, model_validator from pydantic.alias_generators import to_camel from pydantic_settings import BaseSettings @@ -372,6 +372,18 @@ class ProviderConfig(Base): # one place that resolves which of the two shapes (or Gemini's # `api_key_list`) is in effect. endpoints: list[ProviderEndpoint] = Field(default_factory=list) + + @field_validator("endpoints") + @classmethod + def _unique_endpoint_labels(cls, value: list[ProviderEndpoint]) -> list[ProviderEndpoint]: + """Reject a duplicate label -- see the class docstring for why one must be unique.""" + seen: set[str] = set() + for ep in value: + if ep.label in seen: + raise ValueError(f"duplicate endpoint label {ep.label!r}: labels must be unique within a provider") + seen.add(ep.label) + return value + # How requests spread across `endpoints` when there is more than one: # "sticky" keeps using the first healthy entry until it fails, "round_robin" # cycles through all of them. Meaningless with zero or one endpoint. diff --git a/raven/config/update_providers.py b/raven/config/update_providers.py index e8c34c44..cda61000 100644 --- a/raven/config/update_providers.py +++ b/raven/config/update_providers.py @@ -422,6 +422,22 @@ def _redact(value: Any) -> Any: return "****set****" +def _redact_nested_model(instance: BaseModel) -> BaseModel: + """Redact this model's own secret fields, by the same rule as the flat ones. + + ``_flatten_instance`` only recurses into ``BaseModel`` fields, not into a + ``list[BaseModel]`` field like ``ProviderConfig.endpoints`` -- so a caller + that walks ``specs`` (field-name keyed) never sees a per-endpoint field and + can't redact it. This is applied to each list element instead. + """ + updates = { + fname: _redact(getattr(instance, fname)) + for fname, finfo in type(instance).model_fields.items() + if _is_secret_field(fname, finfo) + } + return instance.model_copy(update=updates) if updates else instance + + #: Copilot's credentials are two files LiteLLM owns, not one: the device-flow #: access token and the short-lived API key it is exchanged for. _COPILOT_TOKEN_FILES = ("access-token", "api-key.json") @@ -567,6 +583,7 @@ def list_providers(*, config_path: Path | None = None) -> list[dict[str, Any]]: api_key = getattr(instance, "api_key", "") or "" api_base = getattr(instance, "api_base", None) api_key_list = list(getattr(instance, "api_key_list", []) or []) + endpoints = list(getattr(instance, "endpoints", []) or []) # One rule for every gate: this used to accept a Gemini section holding # only `api_key_list` that routing then skipped and startup refused. @@ -577,8 +594,12 @@ def list_providers(*, config_path: Path | None = None) -> list[dict[str, Any]]: api_key_redacted = "OAuth token" if configured else "(empty)" elif is_local: api_key_redacted = "(not needed for local)" if not api_key else "****set****" + elif api_key or api_key_list: + api_key_redacted = "****set****" + elif endpoints: + api_key_redacted = f"****set**** ({len(endpoints)} endpoints)" else: - api_key_redacted = "****set****" if (api_key or api_key_list) else "(empty)" + api_key_redacted = "(empty)" out.append( { @@ -625,6 +646,8 @@ def get_provider_config( val = flat.get(path_key) if redact_secrets and spec["is_secret"]: out[path_key] = _redact(val) + elif redact_secrets and isinstance(val, list) and val and isinstance(val[0], BaseModel): + out[path_key] = [_redact_nested_model(item) for item in val] else: out[path_key] = val return out diff --git a/raven/providers/rates.py b/raven/providers/rates.py index 82c150ce..daae3921 100644 --- a/raven/providers/rates.py +++ b/raven/providers/rates.py @@ -184,6 +184,29 @@ def _try_litellm_rates(model: str, input_tokens: int, output_tokens: int) -> tup return None +def _cache_only_openrouter_models() -> dict[str, dict]: + """Whatever OpenRouter table is already on hand, without a network call. + + For a caller inside object construction or an asyncio event loop -- + ``AgentLoop.__init__`` and a ``/model`` switch, both of which resolve a + context window before there is a request to size. Those callers want + whatever is already on hand: an in-process cache of any age answers, then + an on-disk cache of any age, then an empty table -- the network is never + touched, because a synchronous ``httpx.Client`` there would block startup + or freeze the running event loop for up to 10s. A stale answer only costs + a stale window; a blocked event loop costs the whole turn. + """ + global _OPENROUTER_CACHE, _OPENROUTER_CACHE_TIME + + if _OPENROUTER_CACHE: + return _OPENROUTER_CACHE + disk = model_catalog_cache.load() + if disk is not None: + _OPENROUTER_CACHE, _OPENROUTER_CACHE_TIME = disk + return _OPENROUTER_CACHE + return {} + + def _fetch_openrouter_models(*, allow_fetch: bool = True) -> dict[str, dict]: """Return OpenRouter's model table, fetched live and cached 1h in-process. @@ -191,28 +214,15 @@ def _fetch_openrouter_models(*, allow_fetch: bool = True) -> dict[str, dict]: id. On any network failure, returns the stale cache (or an empty dict) -- pricing must never raise into the cost path. - ``allow_fetch=False`` is for a caller inside object construction or an - asyncio event loop -- ``AgentLoop.__init__`` and a ``/model`` switch, both of - which resolve a context window before there is a request to size. Those - callers want whatever is already on hand: an in-process cache of any age - answers, then an on-disk cache of any age, then an empty table -- the - network is never touched, because a synchronous ``httpx.Client`` there would - block startup or freeze the running event loop for up to 10s. A stale - answer only costs a stale window; a blocked event loop costs the whole - turn. The per-call usage path is the place that still refreshes normally -- - it already runs inside an ``await``, and is where a stale price or window - is supposed to catch up. + ``allow_fetch=False`` delegates to ``_cache_only_openrouter_models``; see + there for what it changes. The per-call usage path is the place that still + refreshes normally -- it already runs inside an ``await``, and is where a + stale price or window is supposed to catch up. """ global _OPENROUTER_CACHE, _OPENROUTER_CACHE_TIME if not allow_fetch: - if _OPENROUTER_CACHE: - return _OPENROUTER_CACHE - disk = model_catalog_cache.load() - if disk is not None: - _OPENROUTER_CACHE, _OPENROUTER_CACHE_TIME = disk - return _OPENROUTER_CACHE - return {} + return _cache_only_openrouter_models() now = time.time() if _OPENROUTER_CACHE and (now - _OPENROUTER_CACHE_TIME) < _OPENROUTER_CACHE_TTL: @@ -386,15 +396,15 @@ def _lookup_openrouter_entry(model: str, *, allow_fetch: bool = True) -> dict | alias, which stays because within OpenRouter's own namespace a bare id names the same model the full one does. - ``allow_fetch`` passes straight through to ``_fetch_openrouter_models``; see - there for what it changes. Called with the default omitted rather than - ``allow_fetch=True`` explicitly, so a test double standing in for the fetch - with the old zero-argument signature still works unchanged. + ``allow_fetch=False`` reaches ``_cache_only_openrouter_models`` directly + rather than ``_fetch_openrouter_models(allow_fetch=False)`` -- the latter is + the name a test double stands in for with the fetch's old zero-argument + signature, and that double does not declare ``allow_fetch``. """ if not model.startswith("openrouter/"): return None key = model.removeprefix("openrouter/") - table = _fetch_openrouter_models() if allow_fetch else _fetch_openrouter_models(allow_fetch=False) + table = _fetch_openrouter_models() if allow_fetch else _cache_only_openrouter_models() entry = table.get(key) if entry is None and "/" in key: entry = table.get(key.split("/", 1)[1]) diff --git a/tests/test_config_schema.py b/tests/test_config_schema.py index 2df6db14..af030ae2 100644 --- a/tests/test_config_schema.py +++ b/tests/test_config_schema.py @@ -84,3 +84,17 @@ def test_endpoint_strategy_accepts_round_robin_with_camel_alias() -> None: def test_endpoint_strategy_rejects_an_unknown_value() -> None: with pytest.raises(ValidationError): ProviderConfig.model_validate({"endpointStrategy": "random"}) + + +def test_duplicate_endpoint_labels_are_rejected() -> None: + with pytest.raises(ValidationError, match="duplicate endpoint label"): + ProviderConfig.model_validate( + {"endpoints": [{"label": "primary", "apiKey": "sk-1"}, {"label": "primary", "apiKey": "sk-2"}]} + ) + + +def test_distinct_endpoint_labels_are_accepted() -> None: + section = ProviderConfig.model_validate( + {"endpoints": [{"label": "primary", "apiKey": "sk-1"}, {"label": "backup", "apiKey": "sk-2"}]} + ) + assert [e.label for e in section.endpoints] == ["primary", "backup"] diff --git a/tests/test_config_update_providers.py b/tests/test_config_update_providers.py index 7ca08cdb..e3d5f4c0 100644 --- a/tests/test_config_update_providers.py +++ b/tests/test_config_update_providers.py @@ -189,6 +189,35 @@ def test_gemini_api_key_list_plaintext_with_redact_false(cfg_path: Path) -> None assert cfg["api_key_list"] == ["k1", "k2"] +def test_get_redacts_api_key_nested_inside_endpoints(cfg_path: Path) -> None: + add_provider_endpoint("openrouter", label="a", api_key="sk-SUPER-SECRET-A", config_path=cfg_path) + add_provider_endpoint("openrouter", label="b", api_key="sk-SUPER-SECRET-B", config_path=cfg_path) + + cfg = get_provider_config("openrouter", config_path=cfg_path) + + assert [ep.api_key for ep in cfg["endpoints"]] == ["****set****", "****set****"] + assert "sk-SUPER-SECRET-A" not in repr(cfg) + assert "sk-SUPER-SECRET-B" not in repr(cfg) + # Non-secret fields on the same endpoint pass through untouched. + assert [ep.label for ep in cfg["endpoints"]] == ["a", "b"] + + +def test_get_endpoints_plaintext_with_redact_false(cfg_path: Path) -> None: + add_provider_endpoint("openrouter", label="a", api_key="sk-SUPER-SECRET-A", config_path=cfg_path) + + cfg = get_provider_config("openrouter", redact_secrets=False, config_path=cfg_path) + + assert cfg["endpoints"][0].api_key == "sk-SUPER-SECRET-A" + + +def test_get_endpoints_empty_key_renders_as_empty(cfg_path: Path) -> None: + add_provider_endpoint("openrouter", label="a", api_key="", config_path=cfg_path) + + cfg = get_provider_config("openrouter", config_path=cfg_path) + + assert cfg["endpoints"][0].api_key == "(empty)" + + # --------------------------------------------------------------------------- # reset_provider # --------------------------------------------------------------------------- @@ -363,6 +392,18 @@ def test_list_reports_every_provider_with_correct_status(cfg_path: Path) -> None assert len(rows) >= 18 +def test_list_reports_endpoints_only_provider_key_state_consistently(cfg_path: Path) -> None: + """No flat ``api_key`` set, only ``endpoints`` -- the key column must not say + ``(empty)`` while the same row's ``configured`` says the credential is present.""" + add_provider_endpoint("openrouter", label="a", api_key="k1", config_path=cfg_path) + add_provider_endpoint("openrouter", label="b", api_key="k2", config_path=cfg_path) + + row = {p["name"]: p for p in list_providers(config_path=cfg_path)}["openrouter"] + + assert row["configured"] is True + assert row["api_key_redacted"] == "****set**** (2 endpoints)" + + # --------------------------------------------------------------------------- # provider_field_specs # --------------------------------------------------------------------------- diff --git a/tests/test_provider_rates.py b/tests/test_provider_rates.py index 8274e6d1..1f64e6c9 100644 --- a/tests/test_provider_rates.py +++ b/tests/test_provider_rates.py @@ -737,3 +737,24 @@ def test_allow_fetch_false_with_nothing_cached_is_none_and_never_hits_the_networ == rates.DEFAULT_CONTEXT_WINDOW_TOKENS ) assert counter["calls"] == 0 + + +def test_allow_fetch_false_never_calls_fetch_with_a_keyword_it_may_not_accept(monkeypatch): + """``resolve_context_window(..., allow_fetch=False)`` must not depend on + ``_fetch_openrouter_models`` declaring ``allow_fetch``. + + ``conftest``'s autouse network guard binds ``rates._fetch_openrouter_models`` + to a zero-argument stub (``lambda: {}``) for every test by default. Calling + that name with ``allow_fetch=False`` -- as the no-fetch branch of + ``_lookup_openrouter_entry`` used to -- raises ``TypeError`` against that + exact stub. This test reproduces the stub shape directly (not via the + fixture, so it stays correct even if the fixture's own lambda changes) and + asserts the no-fetch ladder still answers. + """ + monkeypatch.setattr(rates, "_fetch_openrouter_models", lambda: {}) + + assert resolve_context_window("openrouter/deepseek/deepseek-v4-pro", allow_fetch=False) is None + assert ( + rates.effective_context_window("openrouter/deepseek/deepseek-v4-pro", None, allow_fetch=False) + == rates.DEFAULT_CONTEXT_WINDOW_TOKENS + ) From 102827dddf4aa771486d63fa5b1add04df225978 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 02:20:49 +0800 Subject: [PATCH 37/78] fix(config): stop keyless endpoints reading as set and invalid sections as empty Two ends of the same consistency rule the review's closure pass turned up. A section whose endpoints carry no keys showed "set (N endpoints)" while credential_status called it unconfigured -- the display now requires an actual key. And the endpoint ops swallowed a section's ValidationError into a default instance, so a hand-edited duplicate label -- which already stops Raven from starting -- plus one endpoint add wiped every real endpoint in the section, on exactly the command a user would reach for to fix things. The loader now lets the error out and names the problem. Co-authored-by: Claude (claude-fable-5) --- raven/config/update_providers.py | 15 +++++++---- tests/test_config_update_providers.py | 36 +++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/raven/config/update_providers.py b/raven/config/update_providers.py index cda61000..d83946d8 100644 --- a/raven/config/update_providers.py +++ b/raven/config/update_providers.py @@ -596,7 +596,7 @@ def list_providers(*, config_path: Path | None = None) -> list[dict[str, Any]]: api_key_redacted = "(not needed for local)" if not api_key else "****set****" elif api_key or api_key_list: api_key_redacted = "****set****" - elif endpoints: + elif endpoints and any(ep.api_key for ep in endpoints): api_key_redacted = f"****set**** ({len(endpoints)} endpoints)" else: api_key_redacted = "(empty)" @@ -869,12 +869,17 @@ def remove_provider_model( def _load_provider_endpoints(name: str, data: dict[str, Any]) -> tuple[type, list[ProviderEndpoint]]: + """Deliberately lets ValidationError out instead of falling back to ``cls()``. + + A section that no longer validates (say, a hand-edited duplicate label) + already stops ``Config.model_validate`` -- Raven will not start on it. The + endpoint commands are the user's likeliest self-rescue there, and a swallow + here made them see an empty list and then *write it back*, wiping every + real endpoint in the section. A loud error names the problem instead. + """ cls = _provider_schema_cls(name) section = _raw_section(data, name) - try: - instance = cls.model_validate(section) - except ValidationError: - instance = cls() + instance = cls.model_validate(section) return cls, list(getattr(instance, "endpoints", []) or []) diff --git a/tests/test_config_update_providers.py b/tests/test_config_update_providers.py index e3d5f4c0..9492aeb3 100644 --- a/tests/test_config_update_providers.py +++ b/tests/test_config_update_providers.py @@ -404,6 +404,42 @@ def test_list_reports_endpoints_only_provider_key_state_consistently(cfg_path: P assert row["api_key_redacted"] == "****set**** (2 endpoints)" +def test_list_does_not_call_keyless_endpoints_set(cfg_path: Path) -> None: + """The mirror direction of the consistency rule: endpoints whose keys are + all empty hold no credential, so the key column must not say set while + credential_status says the section is unconfigured.""" + add_provider_endpoint("openrouter", label="a", api_base="https://a.example/v1", config_path=cfg_path) + + row = {p["name"]: p for p in list_providers(config_path=cfg_path)}["openrouter"] + + assert row["api_key_redacted"] == "(empty)" + + +def test_endpoint_ops_refuse_an_invalid_section_instead_of_wiping_it(cfg_path: Path) -> None: + """A section that no longer validates must stop the endpoint commands loudly. + + Swallowing the error made them see an empty list and write it back: a + hand-edited duplicate label -- which already stops Raven from starting -- + plus one `provider endpoint add` erased every real endpoint in the + section, on exactly the command a user would reach for to fix things. + """ + import json + + from pydantic import ValidationError + + add_provider_endpoint("openrouter", label="keep-1", api_key="k1", config_path=cfg_path) + add_provider_endpoint("openrouter", label="keep-2", api_key="k2", config_path=cfg_path) + data = json.loads(cfg_path.read_text()) + data["providers"]["openrouter"]["endpoints"].append({"label": "keep-1", "apiKey": "dup"}) + cfg_path.write_text(json.dumps(data)) + + with pytest.raises(ValidationError): + add_provider_endpoint("openrouter", label="new", api_key="k3", config_path=cfg_path) + + survivors = json.loads(cfg_path.read_text())["providers"]["openrouter"]["endpoints"] + assert [ep["label"] for ep in survivors] == ["keep-1", "keep-2", "keep-1"] + + # --------------------------------------------------------------------------- # provider_field_specs # --------------------------------------------------------------------------- From 018ae5e81f22f5fe50cd75c405cbb017e8b89944 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 11:15:11 +0800 Subject: [PATCH 38/78] fix(*): render an invalid section's error instead of a traceback The endpoint ops letting ValidationError out made the commands loud, but loud meant a bare traceback on the CLI and an unmapped exception on the RPC face. The three endpoint subcommands now render it the way provider set already does, and the RPC methods map it to the same ConfigValidationError shape a bad slug gets. Co-authored-by: Claude (claude-fable-5) --- raven/cli/provider_commands.py | 15 +++++++++++++++ raven/tui_rpc/methods/model.py | 6 +++--- tests/test_cli_provider_commands.py | 14 ++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/raven/cli/provider_commands.py b/raven/cli/provider_commands.py index 5db79a34..453d69da 100644 --- a/raven/cli/provider_commands.py +++ b/raven/cli/provider_commands.py @@ -740,6 +740,8 @@ def endpoint_add_cmd( client -- a provider using OAuth, or Azure OpenAI / OpenAI Codex, refuses to start with any endpoints configured. """ + from pydantic import ValidationError + from raven.config.update_providers import add_provider_endpoint headers = _parse_extra_headers(extra_headers) @@ -754,6 +756,9 @@ def endpoint_add_cmd( except KeyError as exc: console.print(f"[red]✗[/red] {exc}") raise typer.Exit(1) + except ValidationError as exc: + console.print(f"[red]✗ Validation failed:[/red]\n{exc}") + raise typer.Exit(1) console.print(f"[green]✓[/green] {name} endpoint {label!r} saved ({len(endpoints)} total)") @@ -764,6 +769,8 @@ def endpoint_remove_cmd( label: str = typer.Option(..., "--label", help="Label of the endpoint to remove"), ): """Remove one endpoint by ``--label`` (no-op if the label is not present).""" + from pydantic import ValidationError + from raven.config.update_providers import remove_provider_endpoint try: @@ -771,6 +778,9 @@ def endpoint_remove_cmd( except KeyError as exc: console.print(f"[red]✗[/red] {exc}") raise typer.Exit(1) + except ValidationError as exc: + console.print(f"[red]✗ Validation failed:[/red]\n{exc}") + raise typer.Exit(1) console.print(f"[green]✓[/green] {name} endpoint {label!r} removed ({len(endpoints)} remaining)") @@ -780,6 +790,8 @@ def endpoint_list_cmd( name: str = typer.Argument(..., help="Provider name"), ): """List a provider's endpoints. API keys redacted.""" + from pydantic import ValidationError + from raven.config.update_providers import list_provider_endpoints try: @@ -787,6 +799,9 @@ def endpoint_list_cmd( except KeyError as exc: console.print(f"[red]✗[/red] {exc}") raise typer.Exit(1) + except ValidationError as exc: + console.print(f"[red]✗ Validation failed:[/red]\n{exc}") + raise typer.Exit(1) table = Table(title=f"Provider endpoints: {name}") table.add_column("Label", style="cyan", no_wrap=True) diff --git a/raven/tui_rpc/methods/model.py b/raven/tui_rpc/methods/model.py index 11d917eb..965834ce 100644 --- a/raven/tui_rpc/methods/model.py +++ b/raven/tui_rpc/methods/model.py @@ -368,7 +368,7 @@ async def _endpoints_off_loop(slug: str) -> list[dict[str, Any]]: """The provider's endpoint list, api_key redacted, off the event loop.""" try: return await asyncio.to_thread(list_provider_endpoints, slug) - except KeyError as exc: + except (KeyError, ValidationError) as exc: raise ConfigValidationError(str(exc), data={"slug": slug}) from exc @@ -390,7 +390,7 @@ async def model_add_endpoint(params: dict) -> dict: api_key=parsed.api_key, api_base=parsed.api_base, ) - except KeyError as exc: + except (KeyError, ValidationError) as exc: raise ConfigValidationError(str(exc), data={"slug": parsed.slug}) from exc # Re-read rather than redacting what the write returned, so the one place # deciding how a key is masked stays ``list_provider_endpoints``. @@ -401,7 +401,7 @@ async def model_remove_endpoint(params: dict) -> dict: parsed = _parse(ModelRemoveEndpointParams, params) try: await asyncio.to_thread(remove_provider_endpoint, parsed.slug, parsed.label) - except KeyError as exc: + except (KeyError, ValidationError) as exc: raise ConfigValidationError(str(exc), data={"slug": parsed.slug}) from exc return {"endpoints": await _endpoints_off_loop(parsed.slug)} diff --git a/tests/test_cli_provider_commands.py b/tests/test_cli_provider_commands.py index 0ee6949e..1ee7fd51 100644 --- a/tests/test_cli_provider_commands.py +++ b/tests/test_cli_provider_commands.py @@ -924,6 +924,20 @@ def test_endpoint_add_same_label_replaces(tmp_config: Path) -> None: assert section["endpoints"][0]["apiKey"] == "k2" +def test_endpoint_list_renders_a_validation_error_not_a_traceback(tmp_config: Path) -> None: + """A hand-edited invalid section (duplicate label) must come back as the + same rendered failure `provider set` settled on, not a bare traceback.""" + runner.invoke(app, ["provider", "endpoint", "add", "openrouter", "--label", "a", "--api-key", "k1"]) + data = json.loads(tmp_config.read_text(encoding="utf-8")) + data["providers"]["openrouter"]["endpoints"].append({"label": "a", "apiKey": "dup"}) + tmp_config.write_text(json.dumps(data), encoding="utf-8") + + r = runner.invoke(app, ["provider", "endpoint", "list", "openrouter"]) + + assert r.exit_code == 1 + assert "Validation failed" in r.output + + def test_endpoint_add_unknown_provider_exits_1(tmp_config: Path) -> None: r = runner.invoke( app, From 60adc2447a245e1c8577a6549ef8dbacf9aa0aec Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 15:37:04 +0800 Subject: [PATCH 39/78] fix(providers): match 404 as a token and harden the catalog cache read Two review findings verified by execution. The bare "404" needle in the model-unavailable bucket matched the 404 inside "retry after 1404ms", a request id and a character offset -- each burning a fallback model and cooling a healthy endpoint for an error no swap can fix; it now matches as its own token, the same boundary fix prompt_cache documented for "400". And the disk catalog loader called raw.get on whatever JSON parsed, so a file holding [] or null raised AttributeError into the cost path its docstring promises never raises. Also repoints clawbench's provider import at the symbol that exists (the old target predates this branch and made our constructor edit unreachable) and drops the issue-number comments this branch added to tests. Co-authored-by: Claude (claude-fable-5) --- benchmarks/clawbench/stream.py | 2 +- raven/providers/base.py | 20 ++++++++++++++------ raven/providers/model_catalog_cache.py | 4 ++++ tests/test_agent_loop_stream.py | 2 +- tests/test_cli_onboard_commands.py | 4 ++-- tests/test_error_classification.py | 7 +++++++ tests/test_litellm_provider_attribution.py | 2 +- tests/test_provider_auth_method.py | 2 +- tests/test_provider_rates.py | 16 ++++++++++++++++ 9 files changed, 47 insertions(+), 12 deletions(-) diff --git a/benchmarks/clawbench/stream.py b/benchmarks/clawbench/stream.py index 7f88d125..e8cb876a 100644 --- a/benchmarks/clawbench/stream.py +++ b/benchmarks/clawbench/stream.py @@ -98,7 +98,7 @@ def __init__( restrict_to_workspace: bool, ) -> None: from raven.agent.loop import AgentLoop - from raven.cli.commands import _make_provider + from raven.cli._helpers import make_provider as _make_provider from raven.config.loader import load_config, set_config_path from raven.config.raven import ContextConfig from raven.providers.rates import effective_context_window diff --git a/raven/providers/base.py b/raven/providers/base.py index 64da5ca4..b5a9cb7d 100644 --- a/raven/providers/base.py +++ b/raven/providers/base.py @@ -3,6 +3,7 @@ import asyncio import json import random +import re from abc import ABC, abstractmethod from collections.abc import AsyncIterator from dataclasses import dataclass, field, replace @@ -17,6 +18,11 @@ # OpenRouter -> OpenAI, the rest are the set Hermes accumulated across vendors # (agent/error_classifier.py, MIT, see LICENSES/MIT-hermes-agent.txt). # +#: 404 as its own token. A bare substring also matched the 404 inside +#: "retry after 1404ms", a request id and a character offset -- see the +#: model-unavailable bucket in ``_classify``. +_STATUS_404 = re.compile(r"\b404\b") + # Some are ambiguous alone -- "text is not set" says nothing about images -- and # that is safe here because the recovery is a no-op when no tool result actually # carries one, so a false match costs nothing and never retries blind. @@ -455,21 +461,23 @@ def has(*needles: str) -> bool: return ErrorClassification("billing", should_fallback=True) # Model unavailable / not found → no point retrying it; try another model. - # "404" as a substring mirrors the 429/5xx buckets above: a provider - # that embeds the status into a rendered string (azure's non-200 path) - # reaches here with no exception to read a status code from, and a - # route-level body like "Resource not found" names none of the wordier - # markers. + # The status is matched as its own token, not a substring: a provider + # that embeds it into a rendered string (azure's non-200 path) reaches + # here with no exception to read a code from, but the bare substring + # also matched the 404 inside "retry after 1404ms", a request id, and a + # character offset -- each one burning a fallback model and cooling a + # healthy endpoint for an error no swap can fix. Same hazard, same + # boundary fix as prompt_cache's _STATUS_400. if ( status == 404 or "notfounderror" in names + or _STATUS_404.search(msg) or has( "model not found", "does not exist", "no endpoints", "not available", "unavailable", - "404", ) ): return ErrorClassification("model_unavailable", should_fallback=True) diff --git a/raven/providers/model_catalog_cache.py b/raven/providers/model_catalog_cache.py index 7f5d050b..853a4d9d 100644 --- a/raven/providers/model_catalog_cache.py +++ b/raven/providers/model_catalog_cache.py @@ -57,6 +57,10 @@ def load() -> tuple[dict[str, dict], float] | None: raw = json.loads(path.read_text(encoding="utf-8")) except Exception: return None + # Valid JSON is not necessarily a dict: a file holding [] / null / 42 + # made raw.get raise into the cost path this function promises never to. + if not isinstance(raw, dict): + return None if raw.get("version") != CACHE_VERSION: return None models = raw.get("models") diff --git a/tests/test_agent_loop_stream.py b/tests/test_agent_loop_stream.py index 6272e3ef..572da1fd 100644 --- a/tests/test_agent_loop_stream.py +++ b/tests/test_agent_loop_stream.py @@ -306,7 +306,7 @@ async def on_delta(_text: str) -> None: # --------------------------------------------------------------------------- -# Orphan recovery (issue #152) -- backend never emitted a structured +# Orphan recovery -- backend never emitted a structured # reasoning delta, and the accumulated content carries a closing tag with no # opener (the server's prompt template swallowed it). Only fires for a # provider shaped like a parser-less self-hosted backend diff --git a/tests/test_cli_onboard_commands.py b/tests/test_cli_onboard_commands.py index 6777480f..3b1d17fd 100644 --- a/tests/test_cli_onboard_commands.py +++ b/tests/test_cli_onboard_commands.py @@ -449,7 +449,7 @@ def test_onboard_oauth_non_interactive_errors(tmp_env: Path) -> None: @pytest.mark.parametrize("vendor", ["chatgpt", "bedrock", "sagemaker", "vertex_ai", "azure", "cloudflare"]) def test_onboard_non_interactive_bare_key_refused_vendor_errors(tmp_env: Path, vendor: str) -> None: - """A vendor issue #254 identified as unconfigurable by a bare key is + """A vendor the refusal table marks unconfigurable by a bare key is refused before any credentials are written, instead of being sent through the generic single-key branch that would 401 (or, for chatgpt, be silently ignored) at the first call.""" @@ -1945,7 +1945,7 @@ def _verify(name, *a, **kw): def test_step1_bare_key_refused_vendor_rewinds_to_picker( tmp_env: Path, monkeypatch: pytest.MonkeyPatch, stub_verify, stub_step3, capsys: pytest.CaptureFixture ) -> None: - """Picking a vendor issue #254 identified as unconfigurable by a bare key + """Picking a vendor the refusal table marks unconfigurable by a bare key (chatgpt: it authenticates through Raven's own OAuth path instead) prints the reason and rewinds to the picker via the wizard's existing back mechanism, the same one 'Switch provider' uses -- instead of prompting for diff --git a/tests/test_error_classification.py b/tests/test_error_classification.py index 05cb58dd..09bcb9d8 100644 --- a/tests/test_error_classification.py +++ b/tests/test_error_classification.py @@ -100,6 +100,13 @@ def test_classify_follows_cause_chain(): # A rendered azure non-200 body: no exception, no status attribute, # and a route-level 404 text that names none of the wordier markers. ("Azure OpenAI API Error 404: Resource not found", "model_unavailable"), + # The status must match as its own token: each of these carries "404" + # inside a larger number or id, and classifying them model_unavailable + # burned a fallback model and cooled a healthy endpoint for an error + # no swap can fix. + ("Error: retry after 1404ms", "unknown"), + ("upstream error id=req_a404bc7f", "unknown"), + ("invalid JSON at char 4041", "unknown"), ("This model's maximum context length is 8192 tokens", "context_overflow"), ("401 unauthorized: invalid api key", "auth"), ("400 invalid request: bad schema", "invalid_request"), diff --git a/tests/test_litellm_provider_attribution.py b/tests/test_litellm_provider_attribution.py index c11c41d3..a09391c9 100644 --- a/tests/test_litellm_provider_attribution.py +++ b/tests/test_litellm_provider_attribution.py @@ -64,7 +64,7 @@ def test_extra_msg_keys_non_anthropic_preserves_nothing(): assert LiteLLMProvider._extra_msg_keys("gpt-4o", "gpt-4o") == frozenset() -# --- orphan recovery in _parse_response (issue #152, keyless, no live call) --- +# --- orphan recovery in _parse_response (keyless, no live call) --- # A backend run without a reasoning parser swallows the opening tag into its # prompt template and returns bare reasoning text + a lone ``. That # shape only comes from a self-hosted inference server (hosted_vllm / custom / diff --git a/tests/test_provider_auth_method.py b/tests/test_provider_auth_method.py index fec7604f..597d13e9 100644 --- a/tests/test_provider_auth_method.py +++ b/tests/test_provider_auth_method.py @@ -357,7 +357,7 @@ def key_reads(tree: ast.AST) -> list[int]: assert not offenders, "decide configuredness through providers.auth.credential_status: " + ", ".join(offenders) -#: The six vendors issue #254 identified as unconfigurable by a bare key -- +#: The six vendors the refusal table marks unconfigurable by a bare key -- #: each needs credential material the onboarding wizard's generic single-key #: prompt has no field for. _KEY_REFUSED_VENDORS = ("chatgpt", "bedrock", "sagemaker", "vertex_ai", "azure", "cloudflare") diff --git a/tests/test_provider_rates.py b/tests/test_provider_rates.py index 1f64e6c9..eb828c02 100644 --- a/tests/test_provider_rates.py +++ b/tests/test_provider_rates.py @@ -628,6 +628,22 @@ def test_version_mismatch_ignored(monkeypatch, disk_cache): assert json.loads(disk_cache.read_text(encoding="utf-8"))["version"] == model_catalog_cache.CACHE_VERSION +def test_valid_json_non_dict_cache_is_a_miss(monkeypatch, disk_cache): + """A file holding valid JSON that is not a dict ([] / null / 42) is a miss. + + raw.get on a list raised AttributeError straight through + resolve_context_window and token_rates -- the exact raise the loader's + docstring promises never reaches the cost path. + """ + disk_cache.write_text("[]", encoding="utf-8") + counter = _patch_openrouter(monkeypatch, lambda req: _models_response(_DEEPSEEK_MODELS)) + + cost = _rate_cost("openrouter/deepseek/deepseek-v4-pro", 1000, 500) + + assert counter["calls"] == 1 + assert cost == pytest.approx(1000 * _DEEPSEEK_PRICE[0] + 500 * _DEEPSEEK_PRICE[1], rel=1e-9) + + def test_corrupt_disk_degrades_to_network(monkeypatch, disk_cache): """An unparseable cache file degrades to a miss and falls through to network.""" disk_cache.write_text("{ this is not valid json", encoding="utf-8") From f541bf839304cc4a6eb6467b2973b3417600fe0f Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 15:44:48 +0800 Subject: [PATCH 40/78] fix(providers): keep bedrock caching and knn content intact Two review findings on the capability axis, both reproduced by execution. accepts_cache_control returned False the moment neither addressed_to nor the model id resolved to a spec, so Bedrock Claude -- which speaks anthropic cache_control natively, translated to cachePoint -- silently lost cached-read pricing; an unresolvable address now falls through to the keyword guess the pre-redesign expression carried, with suppress() still learning any wrong guess at runtime, while a resolved spec stays exactly as strict as the redesign made it. And knn-routed endpoints borrow provider_name="custom" for its api_base/api_key shape only, yet the orphan-think gate read that name as a self-hosted backend and cut ordinary content at a stray closing tag; the provider now takes an explicit unparsed_reasoning override and the router passes False. Co-authored-by: Claude (claude-fable-5) --- raven/providers/litellm_provider.py | 21 ++++++++++- raven/providers/per_model_provider.py | 10 +++++- raven/providers/prompt_cache.py | 18 ++++++++++ tests/test_litellm_provider_attribution.py | 41 +++++++++++++++++++++- tests/test_per_model_provider.py | 39 ++++++++++++++++++++ tests/test_provider_prompt_cache.py | 24 +++++++++++++ 6 files changed, 150 insertions(+), 3 deletions(-) diff --git a/raven/providers/litellm_provider.py b/raven/providers/litellm_provider.py index b8ac87b8..234108a0 100644 --- a/raven/providers/litellm_provider.py +++ b/raven/providers/litellm_provider.py @@ -114,8 +114,17 @@ def __init__( disable_auto_cache_control: bool = False, extra_body: dict[str, Any] | None = None, model_overrides: dict[str, dict[str, Any]] | None = None, + *, + unparsed_reasoning: bool | None = None, ): super().__init__(api_key, api_base) + # None: derive from the resolved spec, as emits_unparsed_reasoning always + # did. An explicit bool overrides that derivation outright -- for a + # caller that already knows the answer and for which the spec would + # guess wrong, e.g. a per-model routing endpoint built with + # provider_name="custom" for its api_base/api_key shape alone, not + # because the backend behind it is a self-hosted inference server. + self._unparsed_reasoning = unparsed_reasoning self.default_model = default_model self.extra_headers = extra_headers or {} # When a TokenStrategy (e.g. CacheOptimizer) handles cache_control @@ -223,7 +232,15 @@ def can_serve(self, model: str) -> bool: def emits_unparsed_reasoning(self) -> bool: """See ``LLMProvider.emits_unparsed_reasoning``. - ``self._gateway``, when set, already answers this for both shapes it + ``self._unparsed_reasoning``, when set explicitly at construction, wins + outright: it exists for a caller that already knows the answer and for + which the spec-based guess below is wrong -- a per-model routing + endpoint is built with ``provider_name="custom"`` for its api_base / + api_key shape alone, not because the backend behind it is known to be a + self-hosted inference server, so ``custom`` there would falsely claim + every one of its responses. + + Otherwise, ``self._gateway`` already answers this for both shapes it can hold: a real network gateway (OpenRouter, AiHubMix) fronts one of the large hosted vendors below it, so a bare ```` in content is just content; the generic ``custom`` endpoint and a local spec @@ -242,6 +259,8 @@ def emits_unparsed_reasoning(self) -> bool: from; a resolved direct big vendor (anthropic, openai, ...) never produces it behind its own API. """ + if self._unparsed_reasoning is not None: + return self._unparsed_reasoning spec = self._gateway or find_by_name(canonical_provider_name(self._provider_name)) return spec is not None and (spec.is_local or spec.name == "custom") diff --git a/raven/providers/per_model_provider.py b/raven/providers/per_model_provider.py index 8261e611..a6c891ab 100644 --- a/raven/providers/per_model_provider.py +++ b/raven/providers/per_model_provider.py @@ -23,7 +23,14 @@ def _endpoint_provider(endpoint: "ModelEndpoint") -> LiteLLMProvider: ``provider_name="custom"`` selects the generic OpenAI-compatible gateway spec, so the endpoint's own ``api_base`` / ``api_key`` are carried per call - and several endpoints coexist in one process. + and several endpoints coexist in one process. That name is borrowed for its + api_base/api_key shape only, not as a claim about what is behind it -- a + ``knn``-routed endpoint's backend is whatever the routing config points at, + unknowable here, so ``unparsed_reasoning=False`` keeps this endpoint from + being read as the self-hosted inference server ``custom`` also denotes: a + front-loaded big vendor routed this way would otherwise have its ordinary + content cut at a stray ````, and that cost is worse than the rare + miss on a routing target that genuinely emits unparsed reasoning. """ if not endpoint.api_base: # Without an explicit base LiteLLM falls back to OPENAI_BASE_URL (or @@ -36,6 +43,7 @@ def _endpoint_provider(endpoint: "ModelEndpoint") -> LiteLLMProvider: default_model=endpoint.model, provider_name="custom", extra_headers=session_affinity_headers(), + unparsed_reasoning=False, ) diff --git a/raven/providers/prompt_cache.py b/raven/providers/prompt_cache.py index 3a10312a..fc674239 100644 --- a/raven/providers/prompt_cache.py +++ b/raven/providers/prompt_cache.py @@ -52,6 +52,22 @@ def accepts_cache_control(model: str, *, addressed_to: str = "") -> bool: SiliconFlow client reads as Anthropic's wire from the id alone, and that wire has nowhere to put the field. Passing it keeps the answer about the request rather than about the string. + + A ``addressed_to`` naming a provider Raven carries no spec for (Bedrock, + Vertex, a bare LiteLLM passthrough) resolves to nothing, and so does + ``find_by_model`` on an id whose prefix nobody claims -- in both cases there + is no spec to ask, not a spec that said no. Falling through to + ``find_by_keywords`` there guesses the wire's dialect from the model's own + name instead of giving up: Bedrock speaks Anthropic's ``cache_control`` + natively (translated to ``cachePoint`` on the way out) for exactly the ids + that mention "claude", so the guess is right far more often than a blanket + False would be. A guess is what it is, though -- wrong for a model renamed + away from its vendor's naming, or a passthrough that fronts a wire this + guess did not anticipate -- which is why ``suppress`` exists: an upstream + rejection is learned at runtime and this guess never gets a second try for + that model. Resolving *to* a spec, by contrast, is left exactly as strict as + before -- that path is what stopped a gateway forwarding the field to a + vendor that bills it as an unrecognized block instead of refusing it. """ if not model or model in _SUPPRESSED: return False @@ -59,6 +75,8 @@ def accepts_cache_control(model: str, *, addressed_to: str = "") -> bool: from raven.providers.registry import find_by_keywords, find_by_model, find_by_name addressed = find_by_name(addressed_to) if addressed_to else find_by_model(model) + if addressed is None: + addressed = find_by_keywords(model) if addressed is None or not addressed.supports_prompt_caching: return False diff --git a/tests/test_litellm_provider_attribution.py b/tests/test_litellm_provider_attribution.py index a09391c9..c2beff4a 100644 --- a/tests/test_litellm_provider_attribution.py +++ b/tests/test_litellm_provider_attribution.py @@ -7,7 +7,11 @@ from raven.providers.litellm_provider import _ANTHROPIC_EXTRA_KEYS, LiteLLMProvider -def _make_provider(provider_name: str, extra_headers: dict | None = None) -> LiteLLMProvider: +def _make_provider( + provider_name: str, + extra_headers: dict | None = None, + unparsed_reasoning: bool | None = None, +) -> LiteLLMProvider: with ( patch("raven.providers.litellm_provider.litellm"), patch("raven.providers.litellm_provider.LiteLLMProvider._setup_env"), @@ -16,6 +20,7 @@ def _make_provider(provider_name: str, extra_headers: dict | None = None) -> Lit api_key="sk-test", provider_name=provider_name, extra_headers=extra_headers, + unparsed_reasoning=unparsed_reasoning, ) @@ -129,6 +134,40 @@ def test_parse_response_leaves_bare_close_tag_alone_for_an_unresolved_identity() assert result.content == "discussing the tag in my answer" +# --- explicit unparsed_reasoning override (Should-fix 9) --- +# "custom" is one name for two things: the generic self-hosted inference server +# this normalization exists for, and (per_model_provider._endpoint_provider) the +# api_base/api_key shape a knn-routed endpoint borrows without any claim about +# what backend sits behind it. `unparsed_reasoning=None` (the default) keeps +# deriving the answer from the spec exactly as before; an explicit bool wins +# outright, which is the seam that lets the two meanings of "custom" diverge. + + +def test_unparsed_reasoning_defaults_to_the_spec_derived_guess(): + provider = _make_provider("custom") + assert provider.emits_unparsed_reasoning() is True + + +def test_unparsed_reasoning_explicit_false_overrides_a_true_guess(): + provider = _make_provider("custom", unparsed_reasoning=False) + assert provider.emits_unparsed_reasoning() is False + + +def test_unparsed_reasoning_explicit_true_overrides_a_false_guess(): + provider = _make_provider("anthropic", unparsed_reasoning=True) + assert provider.emits_unparsed_reasoning() is True + + +def test_parse_response_respects_an_explicit_false_override(): + provider = _make_provider("custom", unparsed_reasoning=False) + response = _fake_response("discussing the tag in my answer") + + result = provider._parse_response(response) + + assert result.reasoning_content is None + assert result.content == "discussing the tag in my answer" + + def test_parse_response_leaves_bare_close_tag_alone_for_direct_anthropic(): provider = _make_provider("anthropic") response = _fake_response("discussing the tag in my answer") diff --git a/tests/test_per_model_provider.py b/tests/test_per_model_provider.py index 3e890462..0cfa6700 100644 --- a/tests/test_per_model_provider.py +++ b/tests/test_per_model_provider.py @@ -271,6 +271,45 @@ def test_building_knn_endpoints_leaves_the_process_environment_alone(monkeypatch assert provider._by_model["large"].api_key == "KEY-LARGE" +def test_endpoint_providers_are_built_with_unparsed_reasoning_disabled(): + """Should-fix 9 repro: ``_endpoint_provider`` builds every knn-routed + endpoint with ``provider_name="custom"`` for its api_base/api_key shape + alone -- not as a claim that the backend behind it is a self-hosted + inference server without a reasoning parser. Without the explicit + override, a front-loaded big vendor routed this way would have its + ordinary content clipped at a stray ````. + """ + p = _provider() + assert p._by_model["small"].emits_unparsed_reasoning() is False + assert p._by_model["large"].emits_unparsed_reasoning() is False + + +@pytest.mark.asyncio +async def test_routed_endpoint_does_not_cut_ordinary_content_at_a_stray_think_tag(monkeypatch): + """The evaluator's repro: a knn-routed endpoint's ordinary reply happened + to contain a bare ````, and the ``provider_name="custom"``-derived + guess (before ``unparsed_reasoning=False`` was wired in) read that as an + unparsed reasoning leak and cut the reply in half. + """ + + async def fake_acompletion(**kwargs): + message = MagicMock( + content="the widget's hinge broke", + tool_calls=None, + reasoning_content=None, + thinking_blocks=None, + ) + return MagicMock(choices=[MagicMock(message=message, finish_reason="stop")], usage=None) + + monkeypatch.setattr("raven.providers.litellm_provider.acompletion", fake_acompletion) + + p = _provider() + resp = await p.chat(messages=[{"role": "user", "content": "hi"}], model="small") + + assert resp.content == "the widget's hinge broke" + assert resp.reasoning_content is None + + def test_the_custom_spec_declares_no_env_var_to_write() -> None: """States the field the test above depends on, so a change to it fails here with the reason rather than somewhere unrelated.""" diff --git a/tests/test_provider_prompt_cache.py b/tests/test_provider_prompt_cache.py index fccab46e..928183c8 100644 --- a/tests/test_provider_prompt_cache.py +++ b/tests/test_provider_prompt_cache.py @@ -42,6 +42,30 @@ def test_the_answer_is_the_wire_and_the_family_together(model, expected, why): assert prompt_cache.accepts_cache_control(model) is expected, why +def test_bedrock_model_with_no_registry_spec_keeps_its_caching(): + """Blocker 4 repro: Bedrock carries no ProviderSpec at all, so + ``find_by_name("bedrock")`` resolved to nothing and the strict check + returned False outright -- without ever asking whether the model's own + family reads the field. Bedrock translates ``cache_control`` to its own + ``cachePoint`` natively for exactly the ids that mention "claude", so + falling back to ``find_by_keywords`` here is what the pre-refactor + ``find_by_model(model) or find_by_keywords(model)`` expression covered. + """ + model = "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0" + assert prompt_cache.accepts_cache_control(model, addressed_to="bedrock") is True + + +def test_a_custom_endpoint_still_refuses_despite_an_anthropic_shaped_id(): + """The keyword fallback only fires when ``addressed_to`` resolves to + nothing. ``custom`` resolves to a real spec (``supports_prompt_caching`` + is False), so the fallback must not override that strict answer just + because the model id happens to say "claude" -- that strict path is what + stopped a gateway forwarding the field to a vendor that bills it as an + unrecognized block instead of refusing it. + """ + assert prompt_cache.accepts_cache_control("claude-3-opus", addressed_to="custom") is False + + def test_the_measured_regression_is_the_one_that_changed(): """The three answers this rule was measured against on a real machine. From c8e26a77f86d425df8585c9c986291e411ef8e83 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 15:50:19 +0800 Subject: [PATCH 41/78] fix(*): make every credential gate read the shape requests actually use Three review findings, one root: gates that read a different credential shape than the request path. The presence check now mirrors provider_endpoints' precedence -- endpoints set means the flat fields are ignored outright, so a working flat key alongside a keyless endpoint no longer reports a healthy section whose every request sends an empty key. provider test probes the same resolved endpoint the runtime serves with instead of the flat field, ending the "not_configured while requests succeed" split on the feature's own face. And the setup gate iterates the validated instance's field names rather than raw payload keys, so the camelCase spelling save_config itself emits no longer parks a configured azure or copilot install on the setup-required panel. Co-authored-by: Claude (claude-fable-5) --- raven/config/update_providers.py | 25 +++++++++++-- raven/providers/auth.py | 25 ++++++------- raven/providers/endpoints.py | 7 ++++ raven/tui_rpc/methods/setup.py | 11 +++++- tests/test_config_update_providers.py | 53 +++++++++++++++++++++++++++ tests/test_provider_auth_method.py | 42 +++++++++++++++++++++ tests/test_tui_rpc_setup.py | 21 +++++++++++ 7 files changed, 167 insertions(+), 17 deletions(-) diff --git a/raven/config/update_providers.py b/raven/config/update_providers.py index d83946d8..bbd74174 100644 --- a/raven/config/update_providers.py +++ b/raven/config/update_providers.py @@ -30,6 +30,7 @@ from raven.config.loader import get_config_path, read_raw_or_raise from raven.config.schema import ProviderConfig, ProviderEndpoint, ProvidersConfig +from raven.providers.endpoints import provider_endpoints from raven.providers.registry import ( ProviderSpec, canonical_provider_name, @@ -1015,7 +1016,7 @@ def test_provider( try: spec = _provider_spec(name) - cfg = get_provider_config(name, redact_secrets=False, config_path=config_path) + cls = _provider_schema_cls(name) except KeyError as exc: return { "ok": False, @@ -1027,8 +1028,26 @@ def test_provider( "error": str(exc), } - api_key = cfg.get("api_key") or "" - api_base = cfg.get("api_base") or (spec.default_api_base if spec else "") or "" + path = config_path or get_config_path() + data = read_raw_or_raise(path) + raw_section = _raw_section(data, name) + try: + instance = cls.model_validate(raw_section) + except ValidationError: + instance = cls() + + # Same source as the request path (`provider_endpoints`), not a second + # read of the flat fields -- an endpoints-only or `api_key_list` section + # has no usable flat `api_key`, and reading that field here reported + # `not_configured` on a section the runtime could already serve. The first + # endpoint that actually holds a key is the one a request would use; none + # holding one falls back to the first endpoint's address, matching what a + # section with no endpoints at all (a single resolved entry echoing the + # flat fields) already gave local/keyless providers. + endpoints = provider_endpoints(instance) + endpoint = next((ep for ep in endpoints if ep.api_key), endpoints[0] if endpoints else None) + api_key = endpoint.api_key if endpoint else "" + api_base = (endpoint.api_base if endpoint else None) or (spec.default_api_base if spec else "") or "" derived_api_base = False # Before the token fetch below, which asks a question this backend does not diff --git a/raven/providers/auth.py b/raven/providers/auth.py index 3f820a2d..8fe8ffba 100644 --- a/raven/providers/auth.py +++ b/raven/providers/auth.py @@ -131,24 +131,23 @@ def _present(section: Any, name: str) -> bool: Sections reach here as both: the schema object on the routing path, a raw mapping on the display path. - A section holding ``endpoints`` instead of the flat fields is checked the - same way, one level down: if the flat field is unset, any endpoint that has - ``name`` set also counts. A section with several endpoints and a key on - only one of them is exactly as usable as one with a single flat key -- - routing and startup both read the resolved list (``provider_endpoints``), - not the flat field, so a gate that only looked at the flat field would - reject a section its own request path can serve. + Mirrors the precedence ``provider_endpoints`` reads by: ``endpoints`` set + means the flat fields are ignored outright, not merged with them, so a flat + key alongside a keyless endpoint must not count as present -- that flat key + is never the one a request actually sends. Only when ``endpoints`` is empty + does the flat field (and, for a list field like ``api_key_list``, any + element of it) decide the answer. The gate and the reader must agree on + which shape is in effect; see the ``endpoints`` module docstring. """ if section is None: return False + endpoints = section.get("endpoints") if isinstance(section, dict) else getattr(section, "endpoints", None) + if endpoints: + return any(_present(endpoint, name) for endpoint in endpoints) value = section.get(name) if isinstance(section, dict) else getattr(section, name, None) if isinstance(value, (list, tuple)): - if any(bool(v) for v in value): - return True - elif value: - return True - endpoints = section.get("endpoints") if isinstance(section, dict) else getattr(section, "endpoints", None) - return any(_present(endpoint, name) for endpoint in endpoints or []) + return any(bool(v) for v in value) + return bool(value) def _token_present(provider: str) -> bool: diff --git a/raven/providers/endpoints.py b/raven/providers/endpoints.py index d557330e..e7718120 100644 --- a/raven/providers/endpoints.py +++ b/raven/providers/endpoints.py @@ -25,6 +25,13 @@ every existing caller already got from reading the flat fields directly before this module existed, so returning nothing here would just move the "now what" onto each of them instead of answering it once. + +The gate and the reader must answer the same question the same way: +``raven.providers.auth._present``, which decides whether a section is usable +at all, has to mirror this precedence exactly -- ``endpoints`` non-empty means +only the endpoints count, flat fields included, or a section with a healthy +flat key and a keyless endpoint would pass the gate while this function hands +the empty key to every actual request. """ from __future__ import annotations diff --git a/raven/tui_rpc/methods/setup.py b/raven/tui_rpc/methods/setup.py index 8d6b0580..12d6f280 100644 --- a/raven/tui_rpc/methods/setup.py +++ b/raven/tui_rpc/methods/setup.py @@ -92,7 +92,16 @@ def _detect_provider_configured(payload: dict) -> bool: except Exception: sections = None if sections is not None: - for name in providers: + # Iterate the validated instance's own field names, not the raw + # payload's keys: `canonical_provider_name` does not decompose + # camelCase, so a camelCase key like "azureOpenai" -- the shape + # `ProvidersConfig` serializes to -- fails to resolve back to the + # `azure_openai` field it validated into, and `sections.get` on it + # returns None. The declared fields are always snake_case, so + # asking for those by name always resolves. Extra (unspecced) + # sections keep their original payload spelling. + names = set(type(sections).model_fields) | set(sections.model_extra or {}) + for name in names: section = sections.get(name) if section is not None and credential_status(name, section, include_external=True).ok: return True diff --git a/tests/test_config_update_providers.py b/tests/test_config_update_providers.py index 9492aeb3..22b4904d 100644 --- a/tests/test_config_update_providers.py +++ b/tests/test_config_update_providers.py @@ -603,6 +603,59 @@ def test_test_provider_not_configured_when_api_key_empty(cfg_path: Path) -> None assert result["status"] == "not_configured" +def test_test_provider_endpoints_only_section_probes_the_endpoints_key(cfg_path: Path) -> None: + """No flat ``api_key`` at all -- the credential lives only in ``endpoints``. + + The probe used to read ``cfg.get("api_key")`` directly, which is empty for + an endpoints-only section, and reported ``not_configured`` on a section the + runtime could already serve. It must read the same resolved list + (``provider_endpoints``) that a real request does -- including that + request's own ``api_base``, not the vendor default. + """ + add_provider_endpoint( + "openrouter", + label="primary", + api_key="sk-or-endpoint-test", + api_base="https://example-endpoint.test/v1", + config_path=cfg_path, + ) + + seen: dict[str, Any] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["auth"] = request.headers.get("Authorization") + seen["url"] = str(request.url) + return httpx.Response(200, json={"data": [{"id": "m1"}]}) + + result = probe_provider("openrouter", config_path=cfg_path, transport=_mock_transport(handler)) + + assert result["ok"] is True + assert result["status"] == "valid" + assert seen["auth"] == "Bearer sk-or-endpoint-test" + assert seen["url"].startswith("https://example-endpoint.test/v1") + + +def test_test_provider_gemini_api_key_list_section_probes_the_first_key(cfg_path: Path) -> None: + """Same gap, Gemini's shape: a plural ``api_key_list`` and no flat ``api_key``.""" + set_provider_fields( + "gemini", + {"api_key_list": "AIzaTEST1,AIzaTEST2", "api_base": "https://example-gemini.test/v1"}, + config_path=cfg_path, + ) + + seen: dict[str, Any] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["auth"] = request.headers.get("Authorization") + return httpx.Response(200, json={"data": []}) + + result = probe_provider("gemini", config_path=cfg_path, transport=_mock_transport(handler)) + + assert result["ok"] is True + assert result["status"] == "valid" + assert seen["auth"] == "Bearer AIzaTEST1" + + def test_test_provider_oauth_sends_the_stored_token( cfg_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_provider_auth_method.py b/tests/test_provider_auth_method.py index 597d13e9..0ccbf9c0 100644 --- a/tests/test_provider_auth_method.py +++ b/tests/test_provider_auth_method.py @@ -93,6 +93,15 @@ "model": "anthropic/claude-sonnet-5", "section": {"endpoints": [{"label": "primary", "apiKey": ""}, {"label": "backup", "apiKey": ""}]}, }, + "anthropic_flat_key_and_keyless_endpoint": { + "provider": "anthropic", + "model": "anthropic/claude-sonnet-5", + # A healthy flat key alongside an endpoints list whose only entry has + # none. `provider_endpoints` ignores the flat field outright once + # `endpoints` is set, so this section serves an empty key on every + # request -- the gate must say the same, not fall back to the flat key. + "section": {"apiKey": "sk-ant-TEST", "endpoints": [{"label": "primary", "apiKey": ""}]}, + }, } @@ -225,6 +234,39 @@ def test_an_endpoints_list_with_every_key_empty_is_not_configured(tmp_path: Path assert not _startup_says(case, path) +def test_flat_key_does_not_paper_over_a_keyless_endpoint(tmp_path: Path) -> None: + """The reviewer's repro: a good flat key next to an endpoints list whose + entry has none. + + `provider_endpoints` ignores the flat field once `endpoints` is set, so a + request from this section carries an empty key -- every gate that reads + the flat field first before falling through to endpoints would say the + section is configured while the request 401s. The gate must ask the + endpoints list first, exactly as `provider_endpoints` does. + """ + case = SCENARIOS["anthropic_flat_key_and_keyless_endpoint"] + path = _config_file(tmp_path, case) + assert not _display_says(case, path) + assert not _routing_says(case, path) + assert not _startup_says(case, path) + + +def test_credential_status_false_for_flat_key_and_keyless_endpoint() -> None: + from raven.config.schema import ProviderConfig + from raven.providers.auth import credential_status + + section = ProviderConfig.model_validate({"apiKey": "sk-ant-TEST", "endpoints": [{"label": "a", "apiKey": ""}]}) + assert not credential_status("anthropic", section).ok + + +def test_credential_status_true_when_the_endpoint_itself_has_a_key() -> None: + from raven.config.schema import ProviderConfig + from raven.providers.auth import credential_status + + section = ProviderConfig.model_validate({"endpoints": [{"label": "a", "apiKey": "sk-1"}]}) + assert credential_status("anthropic", section).ok + + def test_credential_status_ok_for_an_endpoints_only_section() -> None: from raven.config.schema import ProviderConfig from raven.providers.auth import credential_status diff --git a/tests/test_tui_rpc_setup.py b/tests/test_tui_rpc_setup.py index 759bc33a..74f1699e 100644 --- a/tests/test_tui_rpc_setup.py +++ b/tests/test_tui_rpc_setup.py @@ -160,6 +160,27 @@ async def test_setup_status_registered_via_helper(fake_home: Path) -> None: assert resp["result"]["provider_configured"] is True +def test_camel_case_provider_key_is_recognized_as_configured() -> None: + """A multi-word provider key arrives as the camelCase ``ProvidersConfig`` + serializes to, and the gate must still see it. + + ``canonical_provider_name`` does not split camelCase (it cannot tell + "azureOpenai" from "OpenRouter" by capitals alone), so iterating the raw + payload's keys and asking ``sections.get`` about each one misses a + declared field spelled this way: it validates into ``azure_openai`` fine, + but ``sections.get("azureOpenai")`` then re-derives a mismatched lookup key + and returns None. A configured Azure or Copilot install used to park on + the setup panel for exactly this reason. + """ + import raven.tui_rpc.methods.setup as setup + + payload = { + "agents": {"defaults": {"model": "azure_openai/my-deployment"}}, + "providers": {"azureOpenai": {"apiKey": "az-TEST", "apiBase": "https://x.openai.azure.com"}}, + } + assert setup._detect_provider_configured(payload) is True + + def test_minimax_oauth_is_detected_from_either_spelling_of_the_prefix(monkeypatch) -> None: """The region is read off the model-id prefix, which arrives underscored. From bb2a109aab5a5bf7364c0df105068c258604204b Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 16:05:46 +0800 Subject: [PATCH 42/78] fix(*): keep the event loop and startup free of the window ladder's weight Three review findings on the same theme: the ladder's costs landing on threads that must not pay them. The two async consumers that still resolved the live window inline now push it to a worker thread, so a cold OpenRouter model no longer freezes every stream and RPC for up to 10s. Construction stops importing litellm outright: allow_fetch=False now also means answer-only-from-an-already-imported litellm, and LazyProvider fires an on_built callback (double-checked against the prewarm race) that re-walks the ladder once the background import lands, so startup stays fast and the window still ends up right. And a /model switch finally reaches the builders that sized themselves at construction: refresh_context_window cascades into the curator's trimmer and the consolidator, ending the session-long stale budget the review demonstrated. Co-authored-by: Claude (claude-fable-5) --- raven/agent/loop/main.py | 25 +++- raven/context_engine/assembler.py | 8 ++ raven/context_engine/base.py | 7 ++ raven/context_engine/curator.py | 6 + raven/context_engine/segments/curator.py | 5 + raven/providers/lazy.py | 36 ++++++ raven/providers/rates.py | 24 +++- raven/tui_rpc/methods/session.py | 20 ++- tests/test_agent_loop_lazy_provider.py | 141 ++++++++++++++++++++++ tests/test_agent_loop_usage_sink.py | 34 ++++++ tests/test_context_engine_factory.py | 103 ++++++++++++++++ tests/test_lazy_provider.py | 75 ++++++++++++ tests/test_provider_rates.py | 53 ++++++++ tests/test_tui_rpc_session_init_bundle.py | 95 +++++++++------ 14 files changed, 584 insertions(+), 48 deletions(-) create mode 100644 tests/test_agent_loop_lazy_provider.py diff --git a/raven/agent/loop/main.py b/raven/agent/loop/main.py index ae6e4ac9..b9de2722 100644 --- a/raven/agent/loop/main.py +++ b/raven/agent/loop/main.py @@ -621,6 +621,15 @@ def __init__( self._register_default_tools() self._apply_disabled_tools() + # LazyProvider defers the litellm import behind a background prewarm + # thread (see providers.lazy); the window this constructor just + # resolved above was answered with allow_import=False, so it can be + # wrong until that import lands. Wiring the callback fixes it up in + # place once the real provider is built -- a no-op for any other + # provider, which has no ``on_built`` to set. + if hasattr(provider, "on_built"): + provider.on_built = self.refresh_context_window + def _apply_disabled_tools(self) -> None: """Unregister tools whose names appear in ``tools.disabled_tools``. @@ -713,6 +722,10 @@ def refresh_context_window(self) -> None: must not quietly discard it. Otherwise the ladder is re-walked so a ``/model`` switch picks up the new model's real window instead of keeping the old one's. + + Also the callback ``LazyProvider.on_built`` fires from its prewarm + thread, i.e. off the event loop -- safe because the only write here is + one ``int`` attribute, and the GIL makes that assignment atomic. """ if self._context_window_explicit: return @@ -720,6 +733,13 @@ def refresh_context_window(self) -> None: # loop, so this must not block it on a synchronous network call. See # rates._fetch_openrouter_models. self.context_window_tokens = effective_context_window(self.model, None, allow_fetch=False) + # Cascade into the builders that sized themselves against the window + # at construction (the Curator's trimmer) and the consolidator -- + # both would otherwise keep budgeting against the pre-switch model's + # window for the rest of the session. The consolidator's window is a + # plain attribute (no setter of its own), set directly here. + self.context_engine.set_context_window(self.context_window_tokens) + self.memory_consolidator.context_window_tokens = self.context_window_tokens def _register_default_tools(self) -> None: """Register the default set of tools.""" @@ -1956,7 +1976,10 @@ async def _run_agent_loop( if self._context_window_explicit: context_max = self.context_window_tokens else: - context_max = resolve_context_window(call_model) or 0 + # Off the event loop: allow_fetch=True here can hit the + # network for up to 10s on an OpenRouter model with both + # caches expired. See rates._fetch_openrouter_models. + context_max = await asyncio.to_thread(resolve_context_window, call_model) or 0 context_used = prompt_tokens + completion_tokens usage_sink.clear() usage_sink["prompt_tokens"] = prompt_tokens diff --git a/raven/context_engine/assembler.py b/raven/context_engine/assembler.py index a19bf7f1..f5c3004e 100644 --- a/raven/context_engine/assembler.py +++ b/raven/context_engine/assembler.py @@ -153,6 +153,14 @@ async def after_turn( if hook is not None: await hook(session_key, outcome, usage) + def set_context_window(self, tokens: int) -> None: + # Delegate to any builder that sized itself against the window at + # construction (only the Curator does; seg1-5 carry no budget). + for builder in self._builders: + setter = getattr(builder, "set_context_window", None) + if setter is not None: + setter(tokens) + def _build_user(self, ctx: AssemblyContext) -> dict[str, Any]: """The single structural user message: runtime context + content.""" runtime_ctx = render.build_runtime_context(self._now_fn, ctx.channel, ctx.chat_id) diff --git a/raven/context_engine/base.py b/raven/context_engine/base.py index 1d4bb1f4..fe1c8944 100644 --- a/raven/context_engine/base.py +++ b/raven/context_engine/base.py @@ -184,6 +184,13 @@ async def after_turn( """ return None + def set_context_window(self, tokens: int) -> None: + """Follow a ``/model`` switch: re-budget whichever builders sized + themselves against the window at construction. Default is no-op so + an engine with no such builder need not override it. + """ + return None + __all__ = [ "AssembledPrefix", diff --git a/raven/context_engine/curator.py b/raven/context_engine/curator.py index 44e4f290..a607c7a5 100644 --- a/raven/context_engine/curator.py +++ b/raven/context_engine/curator.py @@ -367,6 +367,12 @@ def set_provider(self, provider: LLMProvider, model: str) -> None: self.model = model self.trimmer.set_provider(provider, model) + def set_context_window(self, tokens: int) -> None: + """Follow a ``/model`` switch: the trimmer must budget against the + new model's window, not the one it was built with.""" + self.context_window_tokens = tokens + self.trimmer.context_window_tokens = tokens + @staticmethod def working_state_segment(working_state: str | None) -> str: """Render segment 6 text (``# Curator Working State``) or ``""``.""" diff --git a/raven/context_engine/segments/curator.py b/raven/context_engine/segments/curator.py index 66e1ac2b..e78c50c1 100644 --- a/raven/context_engine/segments/curator.py +++ b/raven/context_engine/segments/curator.py @@ -96,6 +96,11 @@ def set_provider(self, provider: LLMProvider, model: str) -> None: self.curator_model = self.config.curator_model or model self.assembler.set_provider(provider, model) + def set_context_window(self, tokens: int) -> None: + """Follow a ``/model`` switch down into the assembler it owns.""" + self.context_window_tokens = tokens + self.assembler.set_context_window(tokens) + async def build(self, ctx: AssemblyContext) -> Segment | None: if ctx.prefix is None: raise RuntimeError("CuratorSegmentBuilder requires ctx.prefix (phase B)") diff --git a/raven/providers/lazy.py b/raven/providers/lazy.py index 34c17e8c..f6a519a8 100644 --- a/raven/providers/lazy.py +++ b/raven/providers/lazy.py @@ -13,6 +13,8 @@ from collections.abc import AsyncIterator, Callable from typing import Any +from loguru import logger + from raven.providers.base import GenerationSettings, LLMProvider, LLMResponse, StreamDelta @@ -34,14 +36,48 @@ def __init__( self._initial_endpoint_label = initial_endpoint_label self._provider: LLMProvider | None = None self._lock = threading.Lock() + self._on_built: Callable[[], None] | None = None def _built(self) -> LLMProvider: if self._provider is None: with self._lock: if self._provider is None: self._provider = self._factory() + callback = self._on_built + if callback is not None: + self._invoke_on_built(callback) return self._provider + @property + def on_built(self) -> "Callable[[], None] | None": + """Fired once, right after the real provider finishes building. + + Lets a caller that skipped the real provider's import at construction + (see ``rates._try_litellm_context_window``'s ``allow_import``) correct + a value it answered cheaply once the real thing is on hand -- e.g. + ``AgentLoop.refresh_context_window``, so a window guessed before + LiteLLM was imported gets fixed once prewarm finishes it. + """ + return self._on_built + + @on_built.setter + def on_built(self, callback: "Callable[[], None] | None") -> None: + """Setting this after the build already happened (prewarm can finish + before the constructor gets here) still fires the callback once, + rather than silently missing the one build event there is.""" + with self._lock: + self._on_built = callback + already_built = self._provider is not None + if already_built and callback is not None: + self._invoke_on_built(callback) + + @staticmethod + def _invoke_on_built(callback: Callable[[], None]) -> None: + try: + callback() + except Exception: + logger.debug("LazyProvider.on_built callback raised", exc_info=True) + def prewarm(self) -> None: """Build the real provider in a daemon thread so the ~2-7s litellm import is hidden behind render + user think-time. Safe to race with the first diff --git a/raven/providers/rates.py b/raven/providers/rates.py index daae3921..a5046c0f 100644 --- a/raven/providers/rates.py +++ b/raven/providers/rates.py @@ -20,6 +20,7 @@ from __future__ import annotations import pathlib +import sys import threading import time from functools import lru_cache @@ -483,8 +484,18 @@ def token_rates(model: str, input_tokens: int = 0, output_tokens: int = 0) -> tu ) -def _try_litellm_context_window(model: str) -> int | None: - """LiteLLM's static model metadata -- offline, covers most mapped providers.""" +def _try_litellm_context_window(model: str, *, allow_import: bool = True) -> int | None: + """LiteLLM's static model metadata -- offline, covers most mapped providers. + + ``allow_import=False`` answers only from a LiteLLM already sitting in + ``sys.modules``: importing it costs ~2-7s, and a caller passing this + (``AgentLoop`` construction, before the lazy provider's prewarm thread has + had a chance to import it) wants the cheap tiers only, not to trigger the + same import it is trying to defer. Once LiteLLM is imported the check is + free and the lookup proceeds exactly as with ``allow_import=True``. + """ + if not allow_import and "litellm" not in sys.modules: + return None try: from raven.providers.litellm_setup import import_litellm @@ -519,10 +530,13 @@ def resolve_context_window(model: str, *, allow_fetch: bool = True) -> int | Non shape the next request rather than cost a label. Unknown models return None so the caller keeps its configured default. - ``allow_fetch=False`` passes straight through to the OpenRouter tier; see - ``_fetch_openrouter_models`` for what it changes. + ``allow_fetch=False`` means "answer from what is already on hand": it + passes through to the OpenRouter tier (see ``_fetch_openrouter_models``) + and also tells the LiteLLM tier not to import LiteLLM on this caller's + behalf (see ``_try_litellm_context_window``) -- a caller cheap enough to + pass this is cheap enough not to pay a fresh import either. """ - window = _try_litellm_context_window(model) + window = _try_litellm_context_window(model, allow_import=allow_fetch) if window: return window diff --git a/raven/tui_rpc/methods/session.py b/raven/tui_rpc/methods/session.py index 106a9704..215db680 100644 --- a/raven/tui_rpc/methods/session.py +++ b/raven/tui_rpc/methods/session.py @@ -25,6 +25,7 @@ from __future__ import annotations +import asyncio import os from datetime import datetime from typing import TYPE_CHECKING, Any, Callable @@ -96,7 +97,7 @@ def _enumerate_skills(agent_loop: "AgentLoop | None") -> dict[str, list[str]]: return {source: sorted(names) for source, names in grouped.items()} -def _baseline_usage( +async def _baseline_usage( agent_loop: "AgentLoop | None", config: "Config", ) -> dict[str, Any]: @@ -114,6 +115,11 @@ def _baseline_usage( Cost is the exception: on a subscription there is no per-token figure, so the banner says so rather than opening at $0.00. Zero here read as free until the first turn replaced it, which is the answer this session will never have. + + ``resolve_context_window`` defaults to ``allow_fetch=True``, so a cold + OpenRouter model can reach for a synchronous 10s HTTP call; this handler + runs on the event loop (an RPC method), so that call is pushed to a + thread rather than blocking every other session in flight. """ from raven.providers.rates import is_plan_billed @@ -121,8 +127,10 @@ def _baseline_usage( configured = config.agents.defaults.context_window_tokens if configured: context_max = configured + elif model: + context_max = await asyncio.to_thread(resolve_context_window, model) or 0 else: - context_max = (resolve_context_window(model) if model else None) or 0 + context_max = 0 return { "input": 0, "output": 0, @@ -134,7 +142,7 @@ def _baseline_usage( } -def _default_session_info( +async def _default_session_info( agent_loop: "AgentLoop | None", config: "Config", ) -> dict[str, Any]: @@ -144,7 +152,7 @@ def _default_session_info( zero usage, ``lazy=True``); version is always real (cached at module load). """ model_id = config.agents.defaults.model - usage = _baseline_usage(agent_loop, config) + usage = await _baseline_usage(agent_loop, config) info: dict[str, Any] = { "model": model_id, "model_id": model_id, @@ -247,7 +255,7 @@ async def session_create( session_id = f"tui:{new_chat_id()}" return { "session_id": session_id, - "info": _default_session_info(agent_loop, load_config()), + "info": await _default_session_info(agent_loop, load_config()), } @@ -291,7 +299,7 @@ async def session_resume( """ agent_loop = _safe_invoke_factory(agent_loop_factory) config = load_config() - info = _default_session_info(agent_loop, config) + info = await _default_session_info(agent_loop, config) session_key = params.get("session_id") if session_key: diff --git a/tests/test_agent_loop_lazy_provider.py b/tests/test_agent_loop_lazy_provider.py new file mode 100644 index 00000000..2f4f522f --- /dev/null +++ b/tests/test_agent_loop_lazy_provider.py @@ -0,0 +1,141 @@ +"""AgentLoop <-> LazyProvider wiring (SF11). + +``LazyProvider`` exists so building the real provider -- which imports +litellm, ~2-7s -- happens on a background prewarm thread instead of stalling +``AgentLoop.__init__``. Construction still needs an answer for the model's +context window, and used to get one by asking LiteLLM's static table, which +imports LiteLLM right there on the main thread if it is not loaded yet -- +defeating the whole point of the lazy provider. ``resolve_context_window``'s +``allow_fetch=False`` construction-time tier now also means "and don't import +LiteLLM to answer this" (see ``rates._try_litellm_context_window``'s +``allow_import``), and ``AgentLoop.__init__`` wires ``LazyProvider.on_built`` +to ``refresh_context_window`` so the window self-corrects once prewarm +finishes the import in the background. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +from raven.agent.loop import AgentLoop +from raven.providers.base import GenerationSettings, LLMProvider +from raven.providers.lazy import LazyProvider + + +class _StubProvider(LLMProvider): + api_key = "test" + + def get_default_model(self) -> str: + return "stub" + + async def chat(self, *args, **kwargs): + raise NotImplementedError + + async def chat_with_retry(self, *args, **kwargs): + raise NotImplementedError + + +def _make_lazy(factory=None) -> LazyProvider: + if factory is None: + + def factory(): + raise AssertionError("the factory must not run during AgentLoop construction") + + return LazyProvider(factory, default_model="stub", generation=GenerationSettings()) + + +def _make_loop(tmp_path: Path, provider) -> AgentLoop: + return AgentLoop( + provider=provider, + workspace=tmp_path, + model="stub", + max_iterations=2, + restrict_to_workspace=True, + ) + + +def test_construction_wires_on_built_to_refresh_context_window(tmp_path: Path) -> None: + lazy = _make_lazy() + agent = _make_loop(tmp_path, lazy) + + assert lazy.on_built == agent.refresh_context_window + + +def test_construction_does_not_build_the_lazy_provider(tmp_path: Path) -> None: + """The factory raises if called -- construction must never invoke it.""" + lazy = _make_lazy() + _make_loop(tmp_path, lazy) # must not raise + + +def test_construction_with_a_plain_provider_skips_the_wiring(tmp_path: Path) -> None: + """A provider with no ``on_built`` attribute (every provider but + LazyProvider) must not make construction raise trying to set one.""" + _make_loop(tmp_path, _StubProvider()) # must not raise + + +def test_on_built_firing_after_construction_updates_the_window(tmp_path: Path, monkeypatch) -> None: + """End to end: once the real provider is built, the callback re-walks the + ladder and the corrected window lands on the loop.""" + import raven.agent.loop.main as agent_loop_main + + lazy = _make_lazy(factory=lambda: _StubProvider()) + agent = _make_loop(tmp_path, lazy) + + monkeypatch.setattr(agent_loop_main, "effective_context_window", lambda *a, **k: 4096) + lazy._built() # simulates prewarm's background build completing + + assert agent.context_window_tokens == 4096 + + +def test_agentloop_construction_with_a_lazy_provider_never_imports_litellm() -> None: + """The regression this fixes: resolving the construction-time window used + to import litellm inline (see module docstring). Run in a subprocess for + a guaranteed-clean ``sys.modules`` -- other test files in this session + force-import litellm at collection time (see test_provider_rates.py), + so an in-process check would pass or fail depending on test order. + """ + script = """ +import sys +import tempfile +from pathlib import Path + +from raven.agent.loop import AgentLoop +from raven.providers import model_catalog_cache +from raven.providers.base import GenerationSettings, LLMProvider +from raven.providers.lazy import LazyProvider + + +class _StubProvider(LLMProvider): + def get_default_model(self): + return "openrouter/deepseek/deepseek-v4-pro" + + +assert "litellm" not in sys.modules, "litellm already imported before construction -- test is not isolated" + +with tempfile.TemporaryDirectory() as td: + model_catalog_cache._CACHE_PATH = Path(td) / "model-catalog.json" + lazy = LazyProvider( + factory=lambda: _StubProvider(), + default_model="openrouter/deepseek/deepseek-v4-pro", + generation=GenerationSettings(), + ) + AgentLoop( + provider=lazy, + workspace=Path(td), + model="openrouter/deepseek/deepseek-v4-pro", + max_iterations=2, + restrict_to_workspace=True, + ) + +print("LITELLM_IMPORTED" if "litellm" in sys.modules else "LITELLM_NOT_IMPORTED") +""" + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, result.stderr + assert "LITELLM_NOT_IMPORTED" in result.stdout, result.stdout + result.stderr diff --git a/tests/test_agent_loop_usage_sink.py b/tests/test_agent_loop_usage_sink.py index c9690bd9..7963e59c 100644 --- a/tests/test_agent_loop_usage_sink.py +++ b/tests/test_agent_loop_usage_sink.py @@ -10,6 +10,7 @@ import json import tempfile +import threading import time from pathlib import Path @@ -17,6 +18,7 @@ import pytest from raven.agent.loop import AgentLoop +from raven.agent.loop import main as agent_loop_main from raven.providers import rates from raven.providers.base import LLMProvider, LLMResponse from raven.spine.message import ChatType, Source @@ -184,6 +186,38 @@ async def test_usage_sink_context_max_stays_explicit_over_live_openrouter(worksp assert sink["context_used"] == 1500 +@pytest.mark.asyncio +async def test_usage_sink_context_max_is_resolved_off_the_event_loop_thread(workspace, monkeypatch): + """SF10: this per-call tier defaults to ``allow_fetch=True``, so a cold + OpenRouter model with both caches expired can reach for a synchronous + ~10s HTTP call. ``_run_agent_loop`` runs on the event loop, so that call + must run on a worker thread, not inline.""" + seen: dict[str, threading.Thread] = {} + + def fake_resolve(model: str) -> int: + seen["thread"] = threading.current_thread() + return 99_999 + + monkeypatch.setattr(agent_loop_main, "resolve_context_window", fake_resolve) + + provider = UsageProvider("stub", 1000, 500) + agent = _make_agent(workspace, provider, model="stub", window=None) + sink: dict = {} + + await agent._process_message( + TurnRequest( + origin=Origin.USER, + source=Source(channel="test", chat_id="c1", sender_id="user", chat_type=ChatType.DM), + text="hi", + ), + session_key="s1", + usage_sink=sink, + ) + + assert sink["context_max"] == 99_999 + assert seen["thread"] is not threading.current_thread() + + # --------------------------------------------------------------------------- # # construction-time ladder: _context_window_explicit + refresh_context_window # # --------------------------------------------------------------------------- # diff --git a/tests/test_context_engine_factory.py b/tests/test_context_engine_factory.py index 1d46a5ed..db3e77bd 100644 --- a/tests/test_context_engine_factory.py +++ b/tests/test_context_engine_factory.py @@ -31,6 +31,7 @@ from raven.context_engine import ContextAssembler from raven.context_engine.factory import build_context_engine from raven.context_engine.segments import MemorySegmentBuilder, SkillsSegmentBuilder +from raven.context_engine.segments.curator import CuratorSegmentBuilder from raven.memory_engine.skill_forge import ( EverosSkillSource, HubSkillSource, @@ -114,6 +115,10 @@ def _memory_builder(engine: ContextAssembler) -> MemorySegmentBuilder: return next(b for b in engine._builders if isinstance(b, MemorySegmentBuilder)) +def _curator_builder(engine: ContextAssembler) -> CuratorSegmentBuilder: + return next(b for b in engine._builders if isinstance(b, CuratorSegmentBuilder)) + + # --------------------------------------------------------------------------- # Factory — always builds the assembler # --------------------------------------------------------------------------- @@ -129,6 +134,36 @@ def test_returns_assembler_without_backend(self, tmp_path: Path) -> None: assert _memory_builder(engine)._backend is None +# --------------------------------------------------------------------------- +# SF6: set_context_window cascades down to the Curator's trimmer -- a +# /model switch must not leave it budgeting against the pre-switch window. +# --------------------------------------------------------------------------- + + +class TestSetContextWindow: + def test_engine_cascades_into_the_curator_trimmer(self, tmp_path: Path) -> None: + engine = _build_engine(tmp_path) + curator = _curator_builder(engine) + assert curator.context_window_tokens == 8192 + assert curator.assembler.context_window_tokens == 8192 + assert curator.assembler.trimmer.context_window_tokens == 8192 + + engine.set_context_window(4096) + + assert curator.context_window_tokens == 4096 + assert curator.assembler.context_window_tokens == 4096 + assert curator.assembler.trimmer.context_window_tokens == 4096 + + def test_engine_cascade_does_not_raise_for_builders_without_the_hook(self, tmp_path: Path) -> None: + """seg1-5 carry no budget and have no ``set_context_window`` -- the + cascade must skip them rather than assume every builder has it.""" + engine = _build_engine(tmp_path) + non_curator = [b for b in engine._builders if not isinstance(b, CuratorSegmentBuilder)] + assert non_curator, "fixture should include builders other than the Curator" + + engine.set_context_window(4096) # must not raise + + # --------------------------------------------------------------------------- # SkillForgeRouter assembly — which sources are present # --------------------------------------------------------------------------- @@ -263,3 +298,71 @@ def test_falls_back_to_legacy_when_stash_none(self, tmp_path: Path) -> None: fake_meta.id = "git-resolver" ids = agent._collect_injected_skill_ids([fake_meta]) assert "local/git-resolver" in ids + + +# --------------------------------------------------------------------------- +# SF6: AgentLoop.refresh_context_window must cascade into the Curator's +# trimmer and the MemoryConsolidator, not just AgentLoop.context_window_tokens +# -- both are built once at construction and hold a snapshot int. +# --------------------------------------------------------------------------- + + +class TestRefreshContextWindowCascade: + def test_model_switch_updates_curator_trimmer_and_consolidator(self, tmp_path: Path, monkeypatch) -> None: + import raven.agent.loop.main as agent_loop_main + + windows = {"stub": 8192, "other-model": 4096} + monkeypatch.setattr( + agent_loop_main, + "effective_context_window", + lambda model, configured, allow_fetch=True: windows[model], + ) + + agent = _make_loop(tmp_path, backend=None) + assert agent.context_window_tokens == 8192 + + curator = _curator_builder(agent.context_engine) + assert curator.context_window_tokens == 8192 + assert curator.assembler.trimmer.context_window_tokens == 8192 + assert agent.memory_consolidator.context_window_tokens == 8192 + + agent.model = "other-model" + agent.refresh_context_window() + + assert agent.context_window_tokens == 4096 + assert curator.context_window_tokens == 4096 + assert curator.assembler.context_window_tokens == 4096 + assert curator.assembler.trimmer.context_window_tokens == 4096 + assert agent.memory_consolidator.context_window_tokens == 4096 + + def test_no_cascade_when_the_window_was_pinned_explicitly(self, tmp_path: Path, monkeypatch) -> None: + """An explicit ``context_window_tokens`` is a deliberate override -- + a later model switch must leave the whole chain untouched.""" + import raven.agent.loop.main as agent_loop_main + + monkeypatch.setattr( + agent_loop_main, + "effective_context_window", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("must not be called when explicit")), + ) + + agent = AgentLoop( + provider=_StubProvider(), + workspace=tmp_path, + model="stub", + max_iterations=2, + restrict_to_workspace=True, + context_window_tokens=8192, + context_config=ContextConfig(), + memory_config=MemoryConfig(), + skill_forge_router_config=SkillForgeRouterConfig(), + ) + curator = _curator_builder(agent.context_engine) + + agent.model = "other-model" + agent.refresh_context_window() + + assert agent.context_window_tokens == 8192 + assert curator.context_window_tokens == 8192 + assert curator.assembler.trimmer.context_window_tokens == 8192 + assert agent.memory_consolidator.context_window_tokens == 8192 diff --git a/tests/test_lazy_provider.py b/tests/test_lazy_provider.py index 6656e538..f4518d3e 100644 --- a/tests/test_lazy_provider.py +++ b/tests/test_lazy_provider.py @@ -176,3 +176,78 @@ def factory(): # the error surfaces on a real call instead with pytest.raises(RuntimeError, match="boom"): asyncio.run(lp.chat([])) + + +# --------------------------------------------------------------------------- # +# on_built -- SF11's fix-up hook: a caller that answered a context-window # +# question cheaply (no import) before the real provider existed gets one # +# chance to correct it once the real provider is on hand. # +# --------------------------------------------------------------------------- # + + +def test_on_built_defaults_to_none() -> None: + lp = LazyProvider(lambda: _FakeProvider(), "cfg-model", GenerationSettings()) + assert lp.on_built is None + + +def test_on_built_fires_once_after_the_first_real_call_builds_it() -> None: + calls: list = [] + lp = LazyProvider(lambda: _FakeProvider(), "cfg-model", GenerationSettings()) + lp.on_built = lambda: calls.append("fired") + + assert calls == [] # setting it does not itself build + asyncio.run(lp.chat([])) + + assert calls == ["fired"] + + +def test_on_built_fires_immediately_when_set_after_the_build_already_happened() -> None: + """prewarm can finish importing before the constructor gets around to + wiring the callback -- the setter must not miss the one build event + there is just because it arrived late.""" + lp = LazyProvider(lambda: _FakeProvider(), "cfg-model", GenerationSettings()) + asyncio.run(lp.chat([])) # materializes _provider before on_built is set + + calls: list = [] + lp.on_built = lambda: calls.append("fired") + + assert calls == ["fired"] + + +def test_on_built_does_not_fire_twice_for_one_build() -> None: + calls: list = [] + lp = LazyProvider(lambda: _FakeProvider(), "cfg-model", GenerationSettings()) + lp.on_built = lambda: calls.append("fired") + + asyncio.run(lp.chat([])) + asyncio.run(lp.chat([])) # second call reuses the memoized provider + + assert calls == ["fired"] + + +def test_on_built_exception_is_swallowed_and_does_not_break_the_build() -> None: + def bad_callback() -> None: + raise RuntimeError("callback boom") + + lp = LazyProvider(lambda: _FakeProvider(), "cfg-model", GenerationSettings()) + lp.on_built = bad_callback + + # must not raise, and the build result must still be usable + assert asyncio.run(lp.chat([])) == "chat" + + +def test_on_built_fires_from_the_prewarm_thread_not_the_caller() -> None: + fired = threading.Event() + seen: dict[str, threading.Thread] = {} + + def on_built() -> None: + seen["thread"] = threading.current_thread() + fired.set() + + lp = LazyProvider(lambda: _FakeProvider(), "cfg-model", GenerationSettings()) + lp.on_built = on_built + lp.prewarm() + + assert fired.wait(timeout=2.0), "on_built did not fire after prewarm built the provider" + assert seen["thread"] is not threading.current_thread() + assert seen["thread"].name == "litellm-prewarm" diff --git a/tests/test_provider_rates.py b/tests/test_provider_rates.py index eb828c02..a88e558c 100644 --- a/tests/test_provider_rates.py +++ b/tests/test_provider_rates.py @@ -8,6 +8,7 @@ from __future__ import annotations import json +import sys import time import httpx @@ -548,6 +549,58 @@ def test_the_window_those_families_report_is_the_vendors_own(): assert rates._try_litellm_context_window("minimax-global/MiniMax-M3") == 1_000_000 +# --- allow_import=False: a cheap caller must not pay LiteLLM's ~2-7s import --- +# +# SF11: ``AgentLoop.__init__`` resolves a construction-time window with +# ``allow_fetch=False`` before ``LazyProvider``'s background thread has had a +# chance to import LiteLLM. Reaching for the import here on the main thread +# defeats the whole point of deferring it. ``import_litellm()`` was already +# forced at module load (see the top of this file) so LiteLLM is always +# present in ``sys.modules`` for every other test below -- these two +# temporarily hide that key to exercise the "not yet imported" branch. + + +def test_try_litellm_context_window_allow_import_false_skips_the_import_when_absent(monkeypatch): + monkeypatch.delitem(sys.modules, "litellm", raising=False) + called = {"n": 0} + real_import_litellm = import_litellm + + def _spy(): + called["n"] += 1 + return real_import_litellm() + + monkeypatch.setattr("raven.providers.litellm_setup.import_litellm", _spy) + + assert rates._try_litellm_context_window("openai-codex/gpt-5.3-codex", allow_import=False) is None + assert called["n"] == 0 + + +def test_try_litellm_context_window_allow_import_false_still_answers_once_imported(): + """Once LiteLLM is already imported the gate is free, and the answer must + not differ from the ``allow_import=True`` (default) path.""" + assert "litellm" in sys.modules + assert rates._try_litellm_context_window( + "openai-codex/gpt-5.3-codex", allow_import=False + ) == rates._try_litellm_context_window("openai-codex/gpt-5.3-codex") + + +def test_resolve_context_window_allow_fetch_false_also_forwards_allow_import_false(monkeypatch): + """One flag, one layer of semantics: allow_fetch=False must reach the + LiteLLM tier as allow_import=False, not just the OpenRouter tier.""" + seen = {} + + def _fake_litellm_tier(model, *, allow_import=True): + seen["allow_import"] = allow_import + return None + + monkeypatch.setattr(rates, "_try_litellm_context_window", _fake_litellm_tier) + monkeypatch.setattr(rates, "_lookup_openrouter_entry", lambda model, *, allow_fetch=True: None) + + rates.resolve_context_window("openrouter/deepseek/deepseek-v4-pro", allow_fetch=False) + + assert seen["allow_import"] is False + + # --- Disk persistence of the OpenRouter catalog --- _DEEPSEEK_PRICE = (0.0000005, 0.0000015) diff --git a/tests/test_tui_rpc_session_init_bundle.py b/tests/test_tui_rpc_session_init_bundle.py index 2d3c052e..9da375b3 100644 --- a/tests/test_tui_rpc_session_init_bundle.py +++ b/tests/test_tui_rpc_session_init_bundle.py @@ -16,6 +16,7 @@ import importlib.metadata import inspect +import threading from pathlib import Path from typing import Any @@ -116,9 +117,9 @@ def config(tmp_path): # --------------------------------------------------------------------------- -def test_default_session_info_contains_real_tools(fake_agent_loop, config) -> None: +async def test_default_session_info_contains_real_tools(fake_agent_loop, config) -> None: """T1.1.a (AC-1): ``info.tools`` carries a real builtin bucket from agent_loop.tools.""" - info = _default_session_info(fake_agent_loop, config) + info = await _default_session_info(fake_agent_loop, config) assert isinstance(info["tools"], dict), "info.tools must be dict[str, list[str]]" assert "builtin" in info["tools"], "info.tools must have a 'builtin' bucket (handoff §3.4 lock)" assert len(info["tools"]["builtin"]) >= 1, "builtin tools list must be non-empty" @@ -128,9 +129,9 @@ def test_default_session_info_contains_real_tools(fake_agent_loop, config) -> No assert info["lazy"] is False, "lazy=False signals tools/skills are real values (vs placeholder True)" -def test_default_session_info_contains_real_skills(fake_agent_loop, config) -> None: +async def test_default_session_info_contains_real_skills(fake_agent_loop, config) -> None: """T1.1.b (AC-2): ``info.skills`` groups skills by SkillMeta.source.""" - info = _default_session_info(fake_agent_loop, config) + info = await _default_session_info(fake_agent_loop, config) assert isinstance(info["skills"], dict), "info.skills must be dict[str, list[str]]" # fake fixture has 2 builtin + 1 workspace assert "builtin" in info["skills"], "fake fixture should produce 'builtin' source group" @@ -144,9 +145,9 @@ def test_default_session_info_contains_real_skills(fake_agent_loop, config) -> N assert len(names) >= 1, f"source group {source!r} has empty list" -def test_default_session_info_contains_real_usage_baseline(fake_agent_loop, config) -> None: +async def test_default_session_info_contains_real_usage_baseline(fake_agent_loop, config) -> None: """T1.1.c (AC-3): ``info.usage`` carries boot baseline (zeros + context_max).""" - info = _default_session_info(fake_agent_loop, config) + info = await _default_session_info(fake_agent_loop, config) usage = info["usage"] assert isinstance(usage, dict) # boot-time: no turn run yet @@ -168,7 +169,7 @@ def test_default_session_info_contains_real_usage_baseline(fake_agent_loop, conf ("anthropic/claude-sonnet-4-5", 0.0), ], ) -def test_the_boot_banner_does_not_open_a_subscription_at_zero( +async def test_the_boot_banner_does_not_open_a_subscription_at_zero( model: str, expected: float | None, fake_agent_loop, @@ -182,14 +183,14 @@ def test_the_boot_banner_does_not_open_a_subscription_at_zero( """ fake_agent_loop.model = model - usage = _default_session_info(fake_agent_loop, config)["usage"] + usage = (await _default_session_info(fake_agent_loop, config))["usage"] assert usage["cost_usd"] == expected -def test_default_session_info_contains_real_version(fake_agent_loop, config) -> None: +async def test_default_session_info_contains_real_version(fake_agent_loop, config) -> None: """T1.1.d (AC-4): ``info.version`` reads importlib.metadata, not hardcoded '0.1'.""" - info = _default_session_info(fake_agent_loop, config) + info = await _default_session_info(fake_agent_loop, config) expected_version = importlib.metadata.version("raven") assert info["version"] == expected_version, ( f"info.version must be importlib.metadata.version('raven') = {expected_version!r}" @@ -197,9 +198,9 @@ def test_default_session_info_contains_real_version(fake_agent_loop, config) -> assert info["version"] != "0.1", "the literal '0.1' placeholder must be replaced" -def test_context_window_reads_config_not_hardcoded_200k(fake_agent_loop, config) -> None: +async def test_context_window_reads_config_not_hardcoded_200k(fake_agent_loop, config) -> None: """``info.context_window`` mirrors ``info.usage.context_max``, not a stub 200000.""" - info = _default_session_info(fake_agent_loop, config) + info = await _default_session_info(fake_agent_loop, config) assert info["context_window"] == info["usage"]["context_max"] assert info["context_window"] != 200_000, "the old stub 200000 must be gone" # Sanity check the default is what we expect: None means "figure it out". @@ -208,9 +209,9 @@ def test_context_window_reads_config_not_hardcoded_200k(fake_agent_loop, config) ) -def test_default_session_info_falls_back_when_agent_loop_none(config) -> None: +async def test_default_session_info_falls_back_when_agent_loop_none(config) -> None: """T1.1.g (AC-7): agent_loop=None graceful fallback per D3 — does not raise.""" - info = _default_session_info(None, config) + info = await _default_session_info(None, config) # tools/skills empty (placeholder semantics) assert info["tools"] == {}, "tools must fall back to empty dict when agent_loop is None" assert info["skills"] == {}, "skills must fall back to empty dict when agent_loop is None" @@ -225,13 +226,13 @@ def test_default_session_info_falls_back_when_agent_loop_none(config) -> None: assert info["lazy"] is True, "lazy=True on agent_loop=None fallback signals UI that tools/skills are placeholder" -def test_default_session_info_falls_back_when_no_usage_tracker(fake_agent_loop_no_tracker, config) -> None: +async def test_default_session_info_falls_back_when_no_usage_tracker(fake_agent_loop_no_tracker, config) -> None: """agent_loop present but no UsageTracker registered. Config may default-off token_wise. Should return baseline zeros + context_max from config (not raise). """ - info = _default_session_info(fake_agent_loop_no_tracker, config) + info = await _default_session_info(fake_agent_loop_no_tracker, config) # tools/skills still real (agent_loop present) assert info["tools"] != {} assert info["skills"] != {} @@ -266,14 +267,14 @@ def test_resolve_context_window_helper_removed() -> None: ) -def test_default_session_info_key_set_matches_expected_v030(fake_agent_loop, config) -> None: +async def test_default_session_info_key_set_matches_expected_v030(fake_agent_loop, config) -> None: """wire-shape lock — info dict has exactly the 12 expected keys. Anti-drift gate: adding a new field to the init bundle MUST update this expected set, forcing an explicit spec amendment, until the dict is promoted to an OpenRPC ``SessionInitBundle`` component schema. """ - info = _default_session_info(fake_agent_loop, config) + info = await _default_session_info(fake_agent_loop, config) expected_keys = { # backward-compat / existing "model", @@ -301,9 +302,9 @@ def test_default_session_info_key_set_matches_expected_v030(fake_agent_loop, con # --------------------------------------------------------------------------- -def test_default_session_info_contains_real_model(fake_agent_loop, config) -> None: +async def test_default_session_info_contains_real_model(fake_agent_loop, config) -> None: """info carries model_id/provider from config (not placeholder).""" - info = _default_session_info(fake_agent_loop, config) + info = await _default_session_info(fake_agent_loop, config) assert info["model_id"] == config.agents.defaults.model assert info["provider"] == config.agents.defaults.provider @@ -311,9 +312,9 @@ def test_default_session_info_contains_real_model(fake_agent_loop, config) -> No # NOTE: lazy assertion moved to test_default_session_info_contains_real_tools -def test_default_session_info_backward_compat_model_field(fake_agent_loop, config) -> None: +async def test_default_session_info_backward_compat_model_field(fake_agent_loop, config) -> None: """info.model retained and equals info.model_id.""" - info = _default_session_info(fake_agent_loop, config) + info = await _default_session_info(fake_agent_loop, config) assert "model" in info assert info["model"] == info["model_id"] assert isinstance(info["model"], str) and info["model"] @@ -327,7 +328,29 @@ def test_placeholder_model_constant_removed() -> None: assert '"claude-sonnet-4-6"' not in src -def test_boot_context_max_uses_live_window_for_openrouter(config, monkeypatch) -> None: +async def test_baseline_usage_resolves_the_window_off_the_event_loop_thread(config, monkeypatch) -> None: + """SF10: ``resolve_context_window`` defaults to ``allow_fetch=True``, so a + cold OpenRouter model with both caches expired can reach for a + synchronous ~10s HTTP call. ``_baseline_usage`` runs on the event loop + (an RPC handler), so that call must run on a worker thread, not inline.""" + seen: dict[str, threading.Thread] = {} + + def fake_resolve(model: str) -> int: + seen["thread"] = threading.current_thread() + return 99_999 + + monkeypatch.setattr(session_module, "resolve_context_window", fake_resolve) + + loop = _FakeAgentLoop(with_usage_tracker=True) + loop.model = "openrouter/deepseek/deepseek-v4-pro" + + usage = await session_module._baseline_usage(loop, config) + + assert usage["context_max"] == 99_999 + assert seen["thread"] is not threading.current_thread() + + +async def test_boot_context_max_uses_live_window_for_openrouter(config, monkeypatch) -> None: """For an OpenRouter model LiteLLM lags on, context_max is the live window.""" monkeypatch.setattr( session_module, @@ -338,12 +361,12 @@ def test_boot_context_max_uses_live_window_for_openrouter(config, monkeypatch) - loop = _FakeAgentLoop(with_usage_tracker=True) loop.model = "openrouter/deepseek/deepseek-v4-pro" - info = _default_session_info(loop, config) + info = await _default_session_info(loop, config) assert info["usage"]["context_max"] == 163840 -def test_boot_context_max_pinned_config_wins_over_live_window(config, monkeypatch) -> None: +async def test_boot_context_max_pinned_config_wins_over_live_window(config, monkeypatch) -> None: """A pinned ``context_window_tokens`` answers even when the live window disagrees.""" config.agents.defaults.context_window_tokens = 8192 monkeypatch.setattr(session_module, "resolve_context_window", lambda model: 163840) @@ -351,7 +374,7 @@ def test_boot_context_max_pinned_config_wins_over_live_window(config, monkeypatc loop = _FakeAgentLoop(with_usage_tracker=True) loop.model = "openrouter/deepseek/deepseek-v4-pro" - info = _default_session_info(loop, config) + info = await _default_session_info(loop, config) assert info["usage"]["context_max"] == 8192 @@ -361,7 +384,7 @@ def test_boot_context_max_pinned_config_wins_over_live_window(config, monkeypatc # --------------------------------------------------------------------------- -def test_default_session_info_carries_the_upgrade_nudge(fake_agent_loop, config, monkeypatch) -> None: +async def test_default_session_info_carries_the_upgrade_nudge(fake_agent_loop, config, monkeypatch) -> None: """A pending release surfaces as ``update_available`` / ``update_command``. The TUI status bar reads both fields, so leaving them unpopulated is the @@ -370,17 +393,17 @@ def test_default_session_info_carries_the_upgrade_nudge(fake_agent_loop, config, """ monkeypatch.setattr(session_module, "update_notice", lambda _v: (True, "raven upgrade")) - info = _default_session_info(fake_agent_loop, config) + info = await _default_session_info(fake_agent_loop, config) assert info["update_available"] is True assert info["update_command"] == "raven upgrade" -def test_default_session_info_omits_the_nudge_when_up_to_date(fake_agent_loop, config, monkeypatch) -> None: +async def test_default_session_info_omits_the_nudge_when_up_to_date(fake_agent_loop, config, monkeypatch) -> None: """No pending release means the keys stay absent, not present-and-false.""" monkeypatch.setattr(session_module, "update_notice", lambda _v: None) - info = _default_session_info(fake_agent_loop, config) + info = await _default_session_info(fake_agent_loop, config) assert "update_available" not in info assert "update_command" not in info @@ -411,26 +434,26 @@ def get_default_model(self) -> str: ) -def test_default_session_info_names_the_endpoint_in_use(config) -> None: +async def test_default_session_info_names_the_endpoint_in_use(config) -> None: """A rotor serves several accounts, so which one is answering is a fact the banner has to carry -- naming only the provider makes them indistinguishable.""" loop = _FakeAgentLoop(with_usage_tracker=True) loop.provider = _rotor(["eu", "us"]) - assert _default_session_info(loop, config)["endpoint"] == "eu" + assert (await _default_session_info(loop, config))["endpoint"] == "eu" -def test_default_session_info_endpoint_is_none_for_a_single_endpoint_provider(fake_agent_loop, config) -> None: +async def test_default_session_info_endpoint_is_none_for_a_single_endpoint_provider(fake_agent_loop, config) -> None: """Every provider but the rotor is reached at one address with no label, so the field is present-and-null rather than a borrowed name.""" - assert _default_session_info(fake_agent_loop, config)["endpoint"] is None + assert (await _default_session_info(fake_agent_loop, config))["endpoint"] is None -def test_reading_the_banner_endpoint_does_not_rotate(config) -> None: +async def test_reading_the_banner_endpoint_does_not_rotate(config) -> None: """Under round_robin the order cursor advances per request. Building the banner is not a request, and a getter that moved it would skip an endpoint every time the panel was rendered.""" loop = _FakeAgentLoop(with_usage_tracker=True) loop.provider = _rotor(["eu", "us"], strategy="round_robin") - assert [_default_session_info(loop, config)["endpoint"] for _ in range(3)] == ["eu", "eu", "eu"] + assert [(await _default_session_info(loop, config))["endpoint"] for _ in range(3)] == ["eu", "eu", "eu"] From 704f2fafa16b5199679b8b4aa728a4f9c95de5ee Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 16:12:23 +0800 Subject: [PATCH 43/78] fix(providers): only a real list is the endpoints shape the gate reads The full-suite sweep caught seven tests whose duck-typed config doubles answered the new endpoints-first read with a truthy non-list, sending the gate down the wrong branch. Sections legitimately reach _present as raw mappings and arbitrary objects, so the descent now requires an actual list -- the only shape provider_endpoints reads -- before it takes over from the flat fields. Co-authored-by: Claude (claude-fable-5) --- raven/providers/auth.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/raven/providers/auth.py b/raven/providers/auth.py index 8fe8ffba..28c5f471 100644 --- a/raven/providers/auth.py +++ b/raven/providers/auth.py @@ -142,7 +142,10 @@ def _present(section: Any, name: str) -> bool: if section is None: return False endpoints = section.get("endpoints") if isinstance(section, dict) else getattr(section, "endpoints", None) - if endpoints: + # isinstance, not truthiness: sections reach here as raw mappings and as + # arbitrary duck-typed objects (test doubles included), and only a real + # list is the endpoints shape provider_endpoints reads. + if isinstance(endpoints, (list, tuple)) and endpoints: return any(_present(endpoint, name) for endpoint in endpoints) value = section.get(name) if isinstance(section, dict) else getattr(section, name, None) if isinstance(value, (list, tuple)): From 008635ccf8906e2379ffedf6df2a15bb9fc95839 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 17:19:04 +0800 Subject: [PATCH 44/78] fix(*): close the reviewer's second-round confirmations Five small items from the follow-up review, plus the middle path it proposed for the upgraded-install pin. The cache-control gate falls back to the auto-detected gateway when the constructor gave only an api_base -- the model id alone read anthropic/... as Anthropic's wire while the request travelled through a gateway that does not carry the field. The endpoints refusal is one predicate shared by the factory and both write paths, so a config that cannot boot is refused when written, rendered cleanly on the CLI and mapped on the RPC face. PerModelProvider mirrors the base loop's per-hop guards (can_serve skip, cache-mark strip), covering the hop that lands on a non-LiteLLM fallback which never strips at send time. The rotor logs which endpoints it exhausted before returning the last error. catalog's vendor lookup normalizes the provider spelling the way merge_key does. And a config still carrying the written-out 65536 default now gets one startup warning naming the line to delete -- the stance stays no-rewrite; the warning is what keeps it from failing silently. Co-authored-by: Claude (claude-fable-5) --- raven/agent/loop/main.py | 12 +++++ raven/cli/_helpers.py | 20 ++------ raven/cli/provider_commands.py | 2 +- raven/config/update_providers.py | 10 +++- raven/providers/catalog.py | 11 +++- raven/providers/endpoint_rotor.py | 11 ++++ raven/providers/litellm_provider.py | 11 +++- raven/providers/per_model_provider.py | 23 +++++++-- raven/providers/registry.py | 24 +++++++++ raven/tui_rpc/methods/model.py | 2 +- tests/test_cli_provider_commands.py | 11 ++++ tests/test_config_update_providers.py | 20 ++++++++ tests/test_litellm_provider_attribution.py | 30 +++++++++++ tests/test_per_model_provider.py | 58 +++++++++++++++++++++- tests/test_provider_catalog.py | 16 ++++++ tests/test_provider_endpoint_rotor.py | 33 ++++++++++++ tests/test_tui_rpc_model.py | 9 ++++ 17 files changed, 278 insertions(+), 25 deletions(-) diff --git a/raven/agent/loop/main.py b/raven/agent/loop/main.py index b9de2722..a6bca44b 100644 --- a/raven/agent/loop/main.py +++ b/raven/agent/loop/main.py @@ -383,6 +383,18 @@ def __init__( # real window. ("Explicit", not "pinned" -- Provider Pin is a different # registered term, see CONTEXT.md.) self._context_window_explicit = bool(context_window_tokens) + if context_window_tokens == 65536: + # The retired schema default, which the old bootstrap wrote to + # disk verbatim -- so on upgraded installs this exact value is + # more often a fossil than a choice. The config is deliberately + # not rewritten (a value the user can see in their own file stays + # theirs); this line is what keeps that stance from failing + # silently. + logger.warning( + "contextWindowTokens: 65536 is pinning the context window (the old default, " + "written out by earlier versions); remove the line from config.json to size " + "it from each model's real window" + ) # allow_fetch=False: construction must not block on a synchronous # network call for an OpenRouter model's window -- whatever is already # cached (in-process or on disk, any age) answers instead. See diff --git a/raven/cli/_helpers.py b/raven/cli/_helpers.py index 634f5bc0..a4b94bef 100644 --- a/raven/cli/_helpers.py +++ b/raven/cli/_helpers.py @@ -114,25 +114,15 @@ def make_provider(config: Config): provider_name = config.get_provider_name(model) p = config.get_provider(model) - from raven.providers.registry import find_by_name + from raven.providers.registry import endpoints_unsupported_reason, find_by_name spec = find_by_name(provider_name) if provider_name else None client = spec.client if spec else "" - # codex / minimax_oauth / azure each need more than a key and an address - # (a device-flow token, a deployment path, ...), and an OAuth section - # (spec.is_oauth, e.g. github_copilot -- which has no dedicated client and - # falls to the litellm branch below) connects through one signed-in - # account, not several. `endpoints` is meaningful only for a plain - # API-key vendor reached through litellm, so a section combining it with - # any of these is rejected here rather than silently using just the first - # entry. - if p and p.endpoints and (client in {"codex", "minimax_oauth", "azure"} or (spec is not None and spec.is_oauth)): - raise MissingCredentialsError( - f"{provider_name} does not support multiple endpoints -- remove the `endpoints` " - "field from its config; this provider connects through a single account, not several", - provider=provider_name or "", - ) + if p and p.endpoints: + reason = endpoints_unsupported_reason(provider_name) + if reason: + raise MissingCredentialsError(reason, provider=provider_name or "") if client == "codex": provider = OpenAICodexProvider(default_model=model) diff --git a/raven/cli/provider_commands.py b/raven/cli/provider_commands.py index 453d69da..e999721a 100644 --- a/raven/cli/provider_commands.py +++ b/raven/cli/provider_commands.py @@ -753,7 +753,7 @@ def endpoint_add_cmd( api_base=api_base or None, extra_headers=headers, ) - except KeyError as exc: + except (KeyError, RuntimeError) as exc: console.print(f"[red]✗[/red] {exc}") raise typer.Exit(1) except ValidationError as exc: diff --git a/raven/config/update_providers.py b/raven/config/update_providers.py index bbd74174..9781b3ae 100644 --- a/raven/config/update_providers.py +++ b/raven/config/update_providers.py @@ -34,6 +34,7 @@ from raven.providers.registry import ( ProviderSpec, canonical_provider_name, + endpoints_unsupported_reason, find_by_name, names_same_provider, normalize_provider_name, @@ -900,9 +901,16 @@ def add_provider_endpoint( field, so re-running this with a rotated ``api_key`` is how the rotation gets written. A new label appends. - Returns the new endpoint list. Raises KeyError for an unknown provider. + Returns the new endpoint list. Raises KeyError for an unknown provider, + RuntimeError for one that ``endpoints_unsupported_reason`` rejects (Codex, + MiniMax OAuth, Azure, or any OAuth section) -- the same rejection + ``make_provider`` applies at build time, applied here before the write + rather than left for that later failure to catch. """ name = canonical_provider_name(name) + reason = endpoints_unsupported_reason(name) + if reason: + raise RuntimeError(reason) path = config_path or get_config_path() data = read_raw_or_raise(path) cls, endpoints = _load_provider_endpoints(name, data) diff --git a/raven/providers/catalog.py b/raven/providers/catalog.py index a6e4b30f..d1589f49 100644 --- a/raven/providers/catalog.py +++ b/raven/providers/catalog.py @@ -166,14 +166,21 @@ def _vendor_id(provider: str, model: str) -> str: A stored id names its provider and the snapshot does not repeat that, so the prefix comes off before the lookup -- including a gateway's, whose rows are filed under the upstream vendor's id. + + ``head`` is always normalized (``split_model_id`` runs it through + ``normalize_provider_name``), so ``provider`` must be too before the + fallback comparison -- the same normalization ``wire.merge_key`` applies to + both sides of its own identity check. Comparing raw missed a provider + Raven carries no spec for whenever it was spelled differently from its + model prefix, e.g. hyphenated ``provider`` against an underscored prefix. """ - from raven.providers.registry import find_by_name + from raven.providers.registry import find_by_name, normalize_provider_name from raven.providers.wire import split_model_id spec = find_by_name(provider) head, rest = split_model_id(model or "") if head and spec and head in spec.route_names: return rest - if head and head == provider: + if head and head == normalize_provider_name(provider): return rest return model or "" diff --git a/raven/providers/endpoint_rotor.py b/raven/providers/endpoint_rotor.py index b4695858..ce3846a3 100644 --- a/raven/providers/endpoint_rotor.py +++ b/raven/providers/endpoint_rotor.py @@ -29,6 +29,8 @@ from dataclasses import dataclass, field from typing import Any +from loguru import logger + from raven.providers.base import ErrorClassification, GenerationSettings, LLMProvider, LLMResponse, StreamDelta from raven.providers.endpoints import ResolvedEndpoint @@ -230,6 +232,7 @@ async def _chat_attempt_with_retry( """ order = self._healthy_order() last_response: LLMResponse | None = None + tried: list[tuple[str, str]] = [] for i in order: response = await self._inners[i]._chat_attempt_with_retry( messages=messages, @@ -246,11 +249,19 @@ async def _chat_attempt_with_retry( classification = response.error_classification or self.classify_error(content=response.content) response.error_classification = classification + tried.append((self._endpoints[i].label, classification.category)) last_response = response if not _rotates(classification): return response self._mark_failure(i) + # Every endpoint was tried and every one failed -- the caller only sees + # the last error otherwise, with no way to tell that the others were + # tried too rather than skipped. + logger.warning( + "All endpoints exhausted, returning the last error. Tried: {}", + ", ".join(f"{label} [{category}]" for label, category in tried), + ) return last_response # type: ignore[return-value] # order always non-empty async def chat( diff --git a/raven/providers/litellm_provider.py b/raven/providers/litellm_provider.py index 234108a0..e18721dc 100644 --- a/raven/providers/litellm_provider.py +++ b/raven/providers/litellm_provider.py @@ -270,10 +270,19 @@ def _supports_cache_control(self, model: str) -> bool: Decided by ``providers.prompt_cache``, which the token strategies ask too -- three copies of this question disagreed, and the one here could not have answered for the marks they place. + + The address falls back to the auto-detected gateway when no + ``provider_name`` was given: several production constructors (the + evolver's launch models, the sentinel planner) pass only an + ``api_base``, and answering from the model id alone reads + ``anthropic/claude-...`` as Anthropic's wire while the request actually + travels through whatever gateway that base names -- a wire that may + have nowhere honest to put the field. """ from raven.providers.prompt_cache import accepts_cache_control - return accepts_cache_control(model, addressed_to=self._provider_name) + addressed = self._provider_name or (self._gateway.name if self._gateway else "") + return accepts_cache_control(model, addressed_to=addressed) def _apply_cache_control( self, diff --git a/raven/providers/per_model_provider.py b/raven/providers/per_model_provider.py index a6c891ab..31cf2cb7 100644 --- a/raven/providers/per_model_provider.py +++ b/raven/providers/per_model_provider.py @@ -11,6 +11,8 @@ from collections.abc import AsyncIterator, Sequence from typing import TYPE_CHECKING, Any +from loguru import logger + from raven.providers.base import LLMProvider, LLMResponse, StreamDelta from raven.providers.litellm_provider import LiteLLMProvider, session_affinity_headers @@ -105,13 +107,28 @@ async def chat_with_retry( Continuation between hops mirrors ``LLMProvider.chat_with_retry``: move to the next hop only on an error classified ``should_fallback`` with a hop remaining; otherwise the response is returned as-is. + + The ``can_serve`` skip and the cache_control strip also mirror that + loop (see ``LLMProvider.chat_with_retry``): a per-model sub-provider + picked for a later hop can be just as unable to serve it, or just as + unable to read a cache marker set for the primary model's vendor, as + the single-instance case those guards were written for. """ + from raven.providers import prompt_cache + model_chain = [model, *(fallback_models or [])] response: LLMResponse | None = None for idx, current_model in enumerate(model_chain): - response = await self._pick(current_model).chat_with_retry( - messages, tools, model=current_model, fallback_models=[], **kwargs - ) + sub = self._pick(current_model) + if idx and not sub.can_serve(current_model or ""): + logger.warning( + "Skipping fallback model={} - this provider instance cannot serve it (wrong vendor)", + current_model, + ) + continue + if idx and not prompt_cache.accepts_cache_control(current_model or ""): + messages, tools = prompt_cache.strip(messages, tools) + response = await sub.chat_with_retry(messages, tools, model=current_model, fallback_models=[], **kwargs) if response.finish_reason != "error": return response diff --git a/raven/providers/registry.py b/raven/providers/registry.py index b0bebf4e..8a5faa4e 100644 --- a/raven/providers/registry.py +++ b/raven/providers/registry.py @@ -620,6 +620,30 @@ def credential_kind(provider: str | None) -> str: return CRED_KEY +def endpoints_unsupported_reason(provider_name: str | None) -> str | None: + """Why ``provider_name``'s config cannot carry an ``endpoints`` list, or None + if it can. + + Shared by every path that could write one -- `make_provider` at build time, + `add_provider_endpoint`, and the TUI `/model` picker's ``model.add_endpoint`` + -- so a section rejected at build time is rejected at write time too, + instead of being accepted by the write paths and only failing later when + something tries to build a provider from it. Codex, MiniMax OAuth and Azure + (``client`` set) each connect through one dedicated client and one account, + not several; an OAuth section reached through litellm instead (``is_oauth``, + e.g. github_copilot, which has no dedicated client) is the same shape. + ``endpoints`` is meaningful only for a plain API-key vendor reached through + litellm. + """ + spec = find_by_name(provider_name) if provider_name else None + if spec is None or not (spec.client or spec.is_oauth): + return None + return ( + f"{provider_name} does not support multiple endpoints -- remove the `endpoints` " + "field from its config; this provider connects through a single account, not several" + ) + + def litellm_spelling(name: str | None) -> str: """How LiteLLM spells this vendor, which is the only form usable as a prefix. diff --git a/raven/tui_rpc/methods/model.py b/raven/tui_rpc/methods/model.py index 965834ce..faad42a1 100644 --- a/raven/tui_rpc/methods/model.py +++ b/raven/tui_rpc/methods/model.py @@ -390,7 +390,7 @@ async def model_add_endpoint(params: dict) -> dict: api_key=parsed.api_key, api_base=parsed.api_base, ) - except (KeyError, ValidationError) as exc: + except (KeyError, ValidationError, RuntimeError) as exc: raise ConfigValidationError(str(exc), data={"slug": parsed.slug}) from exc # Re-read rather than redacting what the write returned, so the one place # deciding how a key is masked stays ``list_provider_endpoints``. diff --git a/tests/test_cli_provider_commands.py b/tests/test_cli_provider_commands.py index 1ee7fd51..9c6a59c4 100644 --- a/tests/test_cli_provider_commands.py +++ b/tests/test_cli_provider_commands.py @@ -938,6 +938,17 @@ def test_endpoint_list_renders_a_validation_error_not_a_traceback(tmp_config: Pa assert "Validation failed" in r.output +def test_endpoint_add_on_an_oauth_provider_renders_the_refusal(tmp_config: Path) -> None: + """The write path shares the factory's refusal; the CLI must render it as + the same clean failure a bad provider name gets, not a bare traceback.""" + r = runner.invoke( + app, + ["provider", "endpoint", "add", "github_copilot", "--label", "x", "--api-key", "k"], + ) + assert r.exit_code == 1 + assert "does not support multiple endpoints" in r.output + + def test_endpoint_add_unknown_provider_exits_1(tmp_config: Path) -> None: r = runner.invoke( app, diff --git a/tests/test_config_update_providers.py b/tests/test_config_update_providers.py index 22b4904d..0c6d9cc8 100644 --- a/tests/test_config_update_providers.py +++ b/tests/test_config_update_providers.py @@ -812,6 +812,26 @@ def test_add_provider_endpoint_unknown_provider_raises(cfg_path: Path) -> None: add_provider_endpoint("nonexistent_provider", label="x", api_key="k", config_path=cfg_path) +@pytest.mark.parametrize("provider", ["azure_openai", "github_copilot"]) +def test_add_provider_endpoint_rejects_providers_that_cannot_rotate(provider: str, cfg_path: Path) -> None: + """Azure connects through a dedicated client, github_copilot through OAuth -- + neither takes an ``endpoints`` list. ``make_provider`` already refused this + at build time; the write path must refuse it before ever touching disk, + not accept a section that starts up broken.""" + with pytest.raises(RuntimeError, match="does not support multiple endpoints"): + add_provider_endpoint(provider, label="x", api_key="k", config_path=cfg_path) + + assert not cfg_path.exists() + + +def test_add_provider_endpoint_still_accepts_a_plain_api_key_provider(cfg_path: Path) -> None: + """openrouter is a plain API-key vendor reached through litellm -- the one + shape ``endpoints`` is meaningful for -- and must be unaffected by the + guard above.""" + endpoints = add_provider_endpoint("openrouter", label="primary", api_key="k1", config_path=cfg_path) + assert [e.label for e in endpoints] == ["primary"] + + def test_remove_provider_endpoint(cfg_path: Path) -> None: add_provider_endpoint("openrouter", label="primary", api_key="k1", config_path=cfg_path) add_provider_endpoint("openrouter", label="backup", api_key="k2", config_path=cfg_path) diff --git a/tests/test_litellm_provider_attribution.py b/tests/test_litellm_provider_attribution.py index c2beff4a..7363230b 100644 --- a/tests/test_litellm_provider_attribution.py +++ b/tests/test_litellm_provider_attribution.py @@ -186,3 +186,33 @@ def test_parse_response_leaves_bare_close_tag_alone_behind_a_gateway(): assert result.reasoning_content is None assert result.content == "discussing the tag in my answer" + +# --- cache-control gate: the address is the wire, not the model id --- + + +def _make_base_only_provider(api_base: str | None) -> LiteLLMProvider: + """A constructor shape several production callers use: api_base, no name. + + The evolver's launch models and the sentinel planner both build this way, + so the gate must answer from the auto-detected gateway, not the model id. + """ + with ( + patch("raven.providers.litellm_provider.litellm"), + patch("raven.providers.litellm_provider.LiteLLMProvider._setup_env"), + ): + return LiteLLMProvider(api_key="sk-test", api_base=api_base) + + +def test_cache_gate_asks_the_detected_gateway_when_no_name_was_given(): + """An anthropic model id through a caching-less gateway must not carry + cache_control: the id alone reads as Anthropic's wire, but the request + travels on whatever the api_base names.""" + p = _make_base_only_provider("https://aihubmix.com/v1") + assert p._supports_cache_control("anthropic/claude-sonnet-4-20250514") is False + + +def test_cache_gate_still_allows_a_caching_gateway_and_direct_anthropic(): + direct = _make_base_only_provider(None) + routed = _make_base_only_provider("https://openrouter.ai/api/v1") + assert direct._supports_cache_control("anthropic/claude-sonnet-4-20250514") is True + assert routed._supports_cache_control("anthropic/claude-sonnet-4-20250514") is True diff --git a/tests/test_per_model_provider.py b/tests/test_per_model_provider.py index 0cfa6700..9b770c52 100644 --- a/tests/test_per_model_provider.py +++ b/tests/test_per_model_provider.py @@ -8,9 +8,10 @@ import pytest from raven.config.schema import ModelEndpoint -from raven.providers.base import GenerationSettings, LLMResponse +from raven.providers.base import ErrorClassification, GenerationSettings, LLMResponse from raven.providers.litellm_provider import LiteLLMProvider from raven.providers.per_model_provider import PerModelProvider +from raven.providers.prompt_cache import CACHE_CONTROL def _fallback(): @@ -129,6 +130,61 @@ async def fake_acompletion(**kwargs): assert calls == [("openai/small", "http://a/v1", "KA")] * 4 + [("openai/large", "http://b/v1", "KB")] +@pytest.mark.asyncio +async def test_chat_with_retry_strips_cache_control_on_a_fallback_hop_that_cannot_read_it(): + """Mirrors ``LLMProvider.chat_with_retry``'s own guard: a breakpoint placed + for the primary model's vendor must not reach a fallback hop whose vendor + cannot read it -- here, a hop that resolves to ``fallback`` (e.g. Azure, + which does not strip cache_control itself). Without this guard the marker + is billed as an unrecognized block or refused outright by the second hop's + wire.""" + p = _provider() + error_resp = LLMResponse( + content="boom", + finish_reason="error", + error_classification=ErrorClassification(category="server_error", should_fallback=True), + ) + p._by_model["small"].chat_with_retry = AsyncMock(return_value=error_resp) + + seen: list[dict] = [] + + async def fake_fallback_chat_with_retry(messages, tools=None, **kwargs): + seen.append({"messages": messages, "tools": tools}) + return LLMResponse(content="FB_RESP", finish_reason="stop") + + p._fallback.chat_with_retry = AsyncMock(side_effect=fake_fallback_chat_with_retry) + + messages = [{"role": "system", "content": "sys", "cache_control": CACHE_CONTROL}] + out = await p.chat_with_retry(messages=messages, model="small", fallback_models=["unrecognized-fallback-model"]) + + assert out.content == "FB_RESP" + assert seen[0]["messages"] == [{"role": "system", "content": "sys"}] + + +@pytest.mark.asyncio +async def test_chat_with_retry_skips_a_fallback_hop_its_sub_provider_cannot_serve(): + """Mirrors ``LLMProvider.chat_with_retry``'s ``can_serve`` guard: a + per-model sub-provider picked for a later hop can be just as unable to + serve it as the single-instance case that guard exists for -- e.g. the + hop resolves to a vendor this sub-provider's credentials do not reach.""" + p = _provider() + error_resp = LLMResponse( + content="boom", + finish_reason="error", + error_classification=ErrorClassification(category="server_error", should_fallback=True), + ) + p._by_model["small"].chat_with_retry = AsyncMock(return_value=error_resp) + p._by_model["large"].can_serve = MagicMock(return_value=False) + p._by_model["large"].chat_with_retry = AsyncMock() + + out = await p.chat_with_retry( + messages=[{"role": "user", "content": "hi"}], model="small", fallback_models=["large"] + ) + + p._by_model["large"].chat_with_retry.assert_not_awaited() + assert out is error_resp + + def test_sub_providers_inherit_configured_model_overrides(): # Routed models are served by their own sub-providers, built here rather # than by make_provider -- so the user's overrides have to be pushed down diff --git a/tests/test_provider_catalog.py b/tests/test_provider_catalog.py index fee11d27..c27e5e17 100644 --- a/tests/test_provider_catalog.py +++ b/tests/test_provider_catalog.py @@ -410,6 +410,22 @@ def test_a_stored_id_round_trips_through_describe() -> None: assert describe("anthropic", "anthropic/claude-sonnet-4-6").label == "Claude Sonnet 4.6" +def test_vendor_id_normalizes_provider_spelling_like_merge_key() -> None: + """``_vendor_id``'s fallback branch used to compare ``head == provider`` + with ``head`` already normalized by ``split_model_id`` but ``provider`` + passed through as-is -- unlike ``wire.merge_key``, which normalizes both + sides of the same comparison. A provider spelled with a hyphen against a + model prefix spelled with an underscore diverged: the strip was skipped + and the whole model string came back as if it named no vendor at all. + """ + from raven.providers.catalog import _vendor_id + from raven.providers.wire import merge_key + + provider, model = "nano-gpt", "nano_gpt/DeepSeek-V3" + assert _vendor_id(provider, model) == "DeepSeek-V3" + assert merge_key(provider, model) == "nano_gpt::deepseek-v3" + + def test_what_the_user_states_about_a_model_beats_the_catalogue() -> None: """The user naming their own deployment beats a catalogue that never heard of it. diff --git a/tests/test_provider_endpoint_rotor.py b/tests/test_provider_endpoint_rotor.py index 1878b244..72872189 100644 --- a/tests/test_provider_endpoint_rotor.py +++ b/tests/test_provider_endpoint_rotor.py @@ -309,6 +309,39 @@ async def test_chat_attempt_with_retry_composes_with_base_model_chain_fallback(c assert (e0.chat_calls, e1.chat_calls) == (2, 1) +async def test_all_endpoints_exhausted_logs_a_warning_naming_each_attempt(clock, caplog): + """Without this, exhausting every endpoint hands the caller only the last + endpoint's error -- no way to tell the others were tried and failed too, + rather than skipped.""" + import logging + + from loguru import logger + + e0 = _StubInner( + "e0", + chat_script=[LLMResponse(content="down on e0", finish_reason="error", error_classification=_FALLBACK_FATAL)], + ) + e1 = _StubInner( + "e1", + chat_script=[LLMResponse(content="down on e1", finish_reason="error", error_classification=_FALLBACK_FATAL)], + ) + rotor = _make_rotor([e0, e1], strategy="sticky") + + # Bridge loguru -> stdlib caplog (loguru doesn't write to logging by default) + handler_id = logger.add(lambda msg: logging.getLogger("loguru.bridge").warning(msg), level="WARNING") + try: + with caplog.at_level(logging.WARNING, logger="loguru.bridge"): + resp = await rotor.chat_with_retry(messages=[], model="m", fallback_models=[]) + finally: + logger.remove(handler_id) + + assert resp.content == "down on e1" + text = "\n".join(rec.message for rec in caplog.records) + assert "e0" in text + assert "e1" in text + assert _FALLBACK_FATAL.category in text + + async def test_generation_assigned_after_construction_propagates_to_every_inner(clock): """``make_provider`` builds the rotor, then assigns ``provider.generation = GenerationSettings(...)`` from config -- see ``raven/cli/_helpers.py``. diff --git a/tests/test_tui_rpc_model.py b/tests/test_tui_rpc_model.py index 99d30873..4c080b0c 100644 --- a/tests/test_tui_rpc_model.py +++ b/tests/test_tui_rpc_model.py @@ -329,6 +329,15 @@ async def test_endpoint_handlers_reject_an_unknown_provider(fake_home: Path, cal await call() +@pytest.mark.parametrize("slug", ["azure_openai", "github_copilot"]) +async def test_add_endpoint_rejects_providers_that_cannot_rotate(slug: str, fake_home: Path) -> None: + """Mirrors ``make_provider``'s build-time rejection: a provider that + connects through one dedicated client/account, not several, must be + rejected here too, with a readable message rather than a bare traceback.""" + with pytest.raises(ConfigValidationError, match="does not support multiple endpoints"): + await model_add_endpoint({"slug": slug, "label": "x", "api_key": "k"}) + + async def test_endpoint_handlers_accept_session_id(fake_home: Path) -> None: # The picker passes its session down like it does for every other model.* # call; a strict param model would reject the key otherwise. From e61224dc91977a9d80b7c4ae32d65f3590a8d200 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 17:24:19 +0800 Subject: [PATCH 45/78] chore(providers): restore the blank line ruff format wants Co-authored-by: Claude (claude-fable-5) --- tests/test_litellm_provider_attribution.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_litellm_provider_attribution.py b/tests/test_litellm_provider_attribution.py index 7363230b..f5368da9 100644 --- a/tests/test_litellm_provider_attribution.py +++ b/tests/test_litellm_provider_attribution.py @@ -187,6 +187,7 @@ def test_parse_response_leaves_bare_close_tag_alone_behind_a_gateway(): assert result.reasoning_content is None assert result.content == "discussing the tag in my answer" + # --- cache-control gate: the address is the wire, not the model id --- From 075c6a0fef902772d38d0fceb7521a05ce49c9f3 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 17:41:18 +0800 Subject: [PATCH 46/78] test(providers): skip interpreter teardown in the litellm-import probe The subprocess asserting that AgentLoop construction leaves litellm unimported exits through os._exit once the mid-process state is printed: the construction drags in native libraries whose atexit hooks segfault the teardown on Linux (returncode -11 in CI, clean exit on macOS), and interpreter shutdown was never part of the claim under test. Co-authored-by: Claude (claude-fable-5) --- tests/test_agent_loop_lazy_provider.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_agent_loop_lazy_provider.py b/tests/test_agent_loop_lazy_provider.py index 2f4f522f..69eedb26 100644 --- a/tests/test_agent_loop_lazy_provider.py +++ b/tests/test_agent_loop_lazy_provider.py @@ -129,7 +129,16 @@ def get_default_model(self): restrict_to_workspace=True, ) -print("LITELLM_IMPORTED" if "litellm" in sys.modules else "LITELLM_NOT_IMPORTED") +ok = "litellm" not in sys.modules +print("LITELLM_NOT_IMPORTED" if ok else "LITELLM_IMPORTED", flush=True) + +# The claim under test is the mid-process sys.modules state above; interpreter +# teardown is not part of it, and AgentLoop's construction drags in native +# libraries whose atexit hooks segfault the teardown on Linux (observed as +# returncode -11 in CI while the same script exits 0 on macOS). Skip teardown. +import os + +os._exit(0 if ok else 2) """ result = subprocess.run( [sys.executable, "-c", script], From 7c4253ee24a61c9f4a2fa489c33eba1e0c0bfa27 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 20:45:48 +0800 Subject: [PATCH 47/78] fix(providers): classify azure non-200 at the source, drop the 404 text match A "404" token match in the model-unavailable bucket sat ahead of the 400 bucket, so a 400 whose rendered body happened to embed a 404 ("upstream said 404") classified as model-unavailable and burned a fallback. The token match existed only for azure's non-200 path, which renders the error into a string before classification can read a status code; azure now classifies at that source, where the real status is still available, and the text match is gone. Co-authored-by: Claude (claude-fable-5) --- raven/providers/azure_openai_provider.py | 18 +++++++++- raven/providers/base.py | 21 ++++------- tests/test_azure_openai_provider.py | 46 ++++++++++++++++++++++++ tests/test_error_classification.py | 14 ++++---- 4 files changed, 77 insertions(+), 22 deletions(-) diff --git a/raven/providers/azure_openai_provider.py b/raven/providers/azure_openai_provider.py index 73a7d307..7e419212 100644 --- a/raven/providers/azure_openai_provider.py +++ b/raven/providers/azure_openai_provider.py @@ -15,6 +15,20 @@ _AZURE_MSG_KEYS = frozenset({"role", "content", "tool_calls", "tool_call_id", "name"}) +class _AzureHTTPError(Exception): + """Carries the real status code past the point where it gets rendered into a string. + + ``classify_error`` reads a status code off an exception; a non-200 response + handled here has one (``response.status_code``), but turning it into + ``LLMResponse.content`` loses it unless something exception-shaped carries + it back through, which is what this does. + """ + + def __init__(self, status_code: int, body: str): + super().__init__(f"Azure OpenAI API Error {status_code}: {body}") + self.status_code = status_code + + class AzureOpenAIProvider(LLMProvider): """ Azure OpenAI provider with API version 2024-10-21 compliance. @@ -168,9 +182,11 @@ async def chat( client.post(url, headers=headers, json=payload), self.generation.timeout ) if response.status_code != 200: + exc = _AzureHTTPError(response.status_code, response.text) return LLMResponse( - content=f"Azure OpenAI API Error {response.status_code}: {response.text}", + content=str(exc), finish_reason="error", + error_classification=self.classify_error(exc), ) response_data = response.json() diff --git a/raven/providers/base.py b/raven/providers/base.py index b5a9cb7d..7e38cd55 100644 --- a/raven/providers/base.py +++ b/raven/providers/base.py @@ -3,7 +3,6 @@ import asyncio import json import random -import re from abc import ABC, abstractmethod from collections.abc import AsyncIterator from dataclasses import dataclass, field, replace @@ -18,11 +17,6 @@ # OpenRouter -> OpenAI, the rest are the set Hermes accumulated across vendors # (agent/error_classifier.py, MIT, see LICENSES/MIT-hermes-agent.txt). # -#: 404 as its own token. A bare substring also matched the 404 inside -#: "retry after 1404ms", a request id and a character offset -- see the -#: model-unavailable bucket in ``_classify``. -_STATUS_404 = re.compile(r"\b404\b") - # Some are ambiguous alone -- "text is not set" says nothing about images -- and # that is safe here because the recovery is a no-op when no tool result actually # carries one, so a false match costs nothing and never retries blind. @@ -461,17 +455,16 @@ def has(*needles: str) -> bool: return ErrorClassification("billing", should_fallback=True) # Model unavailable / not found → no point retrying it; try another model. - # The status is matched as its own token, not a substring: a provider - # that embeds it into a rendered string (azure's non-200 path) reaches - # here with no exception to read a code from, but the bare substring - # also matched the 404 inside "retry after 1404ms", a request id, and a - # character offset -- each one burning a fallback model and cooling a - # healthy endpoint for an error no swap can fix. Same hazard, same - # boundary fix as prompt_cache's _STATUS_400. + # No bare "404" substring here: it also matched the 404 inside "retry + # after 1404ms", a request id, and a character offset -- each one + # burning a fallback model and cooling a healthy endpoint for an error + # no swap can fix. A provider that renders its non-200 body into a + # plain string before it reaches this method (azure's path) attaches + # the classification at the source instead, where the real status + # code is still available -- see ``AzureOpenAIProvider.chat``. if ( status == 404 or "notfounderror" in names - or _STATUS_404.search(msg) or has( "model not found", "does not exist", diff --git a/tests/test_azure_openai_provider.py b/tests/test_azure_openai_provider.py index 1563360d..1aba02cc 100644 --- a/tests/test_azure_openai_provider.py +++ b/tests/test_azure_openai_provider.py @@ -32,6 +32,33 @@ async def post(self, *args: Any, **kwargs: Any) -> Any: await asyncio.sleep(10) +class _FakeResponse: + """httpx.Response stand-in carrying just what the non-200 branch reads.""" + + def __init__(self, status_code: int, text: str) -> None: + self.status_code = status_code + self.text = text + + +def _non_ok_client_cls(status_code: int, text: str) -> type: + """Build an httpx.AsyncClient stand-in whose POST returns a fixed non-200 response.""" + + class _Client: + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + + async def __aenter__(self) -> "_Client": + return self + + async def __aexit__(self, *args: Any) -> bool: + return False + + async def post(self, *args: Any, **kwargs: Any) -> _FakeResponse: + return _FakeResponse(status_code, text) + + return _Client + + def _make_provider(timeout: float) -> AzureOpenAIProvider: provider = AzureOpenAIProvider( api_key="test-key", @@ -56,6 +83,25 @@ async def test_chat_wall_clock_cap_returns_classified_error(monkeypatch: pytest. assert resp.error_classification.retryable is True +@pytest.mark.asyncio +async def test_a_rendered_404_body_classifies_as_model_unavailable(monkeypatch: pytest.MonkeyPatch) -> None: + """The status code is real here (``response.status_code``), unlike the + swallowed-string path ``classify_error`` degrades to elsewhere -- so this + is classified from it directly, before the response becomes a string. + """ + monkeypatch.setattr( + "raven.providers.azure_openai_provider.httpx.AsyncClient", + _non_ok_client_cls(404, "Resource not found"), + ) + provider = _make_provider(timeout=5.0) + resp = await provider.chat(messages=[{"role": "user", "content": "hi"}], model="gpt-4o") + assert resp.finish_reason == "error" + assert resp.error_classification is not None + assert resp.error_classification.category == "model_unavailable" + assert resp.error_classification.should_fallback is True + assert "404" in (resp.content or "") + + def test_a_configured_deployment_decides_the_url_path() -> None: """The deployment is a connection parameter, not part of the model id. diff --git a/tests/test_error_classification.py b/tests/test_error_classification.py index 09bcb9d8..6b04565d 100644 --- a/tests/test_error_classification.py +++ b/tests/test_error_classification.py @@ -97,13 +97,13 @@ def test_classify_follows_cause_chain(): ("connection reset by peer", "network"), ("insufficient credit / billing", "billing"), ("model not found", "model_unavailable"), - # A rendered azure non-200 body: no exception, no status attribute, - # and a route-level 404 text that names none of the wordier markers. - ("Azure OpenAI API Error 404: Resource not found", "model_unavailable"), - # The status must match as its own token: each of these carries "404" - # inside a larger number or id, and classifying them model_unavailable - # burned a fallback model and cooled a healthy endpoint for an error - # no swap can fix. + # None of these carries any of the wordier model_unavailable markers, + # even though each one has "404" inside a larger number or id -- + # matching it as a bare substring once burned a fallback model and + # cooled a healthy endpoint for an error no swap could fix. Azure's own + # rendered non-200 body no longer reaches this degraded path at all: + # see ``AzureOpenAIProvider.chat``, which classifies from the live + # status code before the response is turned into a string. ("Error: retry after 1404ms", "unknown"), ("upstream error id=req_a404bc7f", "unknown"), ("invalid JSON at char 4041", "unknown"), From 4633d730b0fda5c106b32c9e2e1b5132543ae43f Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 20:46:28 +0800 Subject: [PATCH 48/78] fix(providers): expose model_overrides through the endpoint rotor PerModelProvider reads model_overrides off its fallback with a getattr default, and EndpointRotorProvider did not define it -- so on exactly the configs with several endpoints to rotate, per-model overrides went missing silently. The rotor now delegates to its first inner, the same one-answer-per-section reasoning as can_serve. The existing test built its fallback as a LiteLLMProvider, the one class that does define the attribute, which is why it stayed green; it is now parametrized over a plain and a rotor-wrapped fallback. Co-authored-by: Claude (claude-fable-5) --- raven/providers/endpoint_rotor.py | 11 +++++++++++ tests/test_per_model_provider.py | 32 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/raven/providers/endpoint_rotor.py b/raven/providers/endpoint_rotor.py index ce3846a3..43051408 100644 --- a/raven/providers/endpoint_rotor.py +++ b/raven/providers/endpoint_rotor.py @@ -349,5 +349,16 @@ def emits_unparsed_reasoning(self) -> bool: of its wire is one answer, not one per endpoint.""" return self._inners[0].emits_unparsed_reasoning() + @property + def model_overrides(self) -> dict[str, dict[str, Any]]: + """Delegates to the first endpoint's inner, same reasoning as ``can_serve``: + every inner was built from this same section, so the overrides are one + answer, not one per endpoint. Without this, ``PerModelProvider``'s + ``getattr(fallback, "model_overrides", None)`` silently read nothing + back whenever ``fallback`` was a rotor -- the shape a multi-endpoint + section builds -- and per-model overrides went missing on exactly the + configs that had several endpoints to rotate.""" + return self._inners[0].model_overrides + def get_default_model(self) -> str: return self._default_model diff --git a/tests/test_per_model_provider.py b/tests/test_per_model_provider.py index 9b770c52..ef15d550 100644 --- a/tests/test_per_model_provider.py +++ b/tests/test_per_model_provider.py @@ -196,6 +196,38 @@ def test_sub_providers_inherit_configured_model_overrides(): assert p._by_model["small"].model_overrides == {"small": {"top_p": 0.3}} +def test_sub_providers_inherit_overrides_from_a_rotor_fallback(): + """``fallback`` in production is whatever ``make_provider`` built, and a + multi-endpoint section builds an ``EndpointRotorProvider`` there, not a + ``LiteLLMProvider`` -- the only class every other test in this module + exercises. ``getattr(fallback, "model_overrides", None)`` silently read + nothing back off a rotor before it gained the delegating property, and a + routed model's overrides went missing on exactly the configs with several + endpoints to rotate. + """ + from raven.providers.endpoint_rotor import EndpointRotorProvider + from raven.providers.endpoints import ResolvedEndpoint + + def make_inner(ep): + return LiteLLMProvider( + api_key=ep.api_key, + api_base=ep.api_base, + default_model="fb", + provider_name="openrouter", + model_overrides={"small": {"top_p": 0.3}}, + ) + + rotor = EndpointRotorProvider( + [ResolvedEndpoint(label="a", api_key="k1", api_base="http://a/v1", extra_headers=None)], + make_inner, + default_model="fb", + ) + + p = PerModelProvider([ModelEndpoint(model="small", api_base="http://a/v1")], fallback=rotor) + + assert p._by_model["small"].model_overrides == {"small": {"top_p": 0.3}} + + def test_sub_providers_go_through_litellm(): p = _provider() assert all(isinstance(sub, LiteLLMProvider) for sub in p._by_model.values()) From 3d04d83f8a8ae030efe8deb2f380dc7942ad0178 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 20:47:02 +0800 Subject: [PATCH 49/78] docs(agent): refresh_context_window docstring matches its write set The cross-thread-safety note said "the only write here is one int attribute" while the body cascades into four int writes across three objects. The safety argument survives (each write is a GIL-atomic int assignment); the count and the objects are now stated as they are. Co-authored-by: Claude (claude-fable-5) --- raven/agent/loop/main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/raven/agent/loop/main.py b/raven/agent/loop/main.py index a6bca44b..c29aba21 100644 --- a/raven/agent/loop/main.py +++ b/raven/agent/loop/main.py @@ -736,8 +736,10 @@ def refresh_context_window(self) -> None: keeping the old one's. Also the callback ``LazyProvider.on_built`` fires from its prewarm - thread, i.e. off the event loop -- safe because the only write here is - one ``int`` attribute, and the GIL makes that assignment atomic. + thread, i.e. off the event loop -- safe because every write this + method triggers, here and in the builders it cascades into (the + Curator and its trimmer, the consolidator), is a plain ``int`` + attribute assignment, and the GIL makes each one atomic. """ if self._context_window_explicit: return From cbe24b88dd0e115a3729d5d6c7d149cd585bb8b2 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 20:49:36 +0800 Subject: [PATCH 50/78] fix(providers): endpoint entries inherit the flat api_base and extra_headers One config shape had three answers: the reader dropped the section's flat address and headers once endpoints were set, the credential gate judged each entry's own fields, and the builder quietly filled the address back in. The natural product of `endpoint add --label --api-key` (entries sharing the section's flat address) was refused at startup by the gate while the builder would have run it. One semantics now, owned by provider_endpoints: an entry naming no api_base or extra_headers of its own inherits the section's flat value, the same way api_key_list entries always shared it; api_key is still never inherited, so a stale flat key cannot outlive the endpoints that replaced it. The gate consumes the resolved list instead of re-deriving the precedence, and the builder keeps only the spec-default fallback for a flat address that is itself empty. Co-authored-by: Claude (claude-fable-5) --- raven/cli/_helpers.py | 13 +++++- raven/config/schema.py | 5 ++- raven/providers/auth.py | 22 ++++++---- raven/providers/endpoints.py | 67 ++++++++++++++++++------------ tests/test_provider_auth_method.py | 33 +++++++++++++++ tests/test_provider_endpoints.py | 59 +++++++++++++++++++++++++- 6 files changed, 158 insertions(+), 41 deletions(-) diff --git a/raven/cli/_helpers.py b/raven/cli/_helpers.py index a4b94bef..17f8f328 100644 --- a/raven/cli/_helpers.py +++ b/raven/cli/_helpers.py @@ -153,9 +153,15 @@ def make_provider(config: Config): def make_inner(ep): return LiteLLMProvider( api_key=ep.api_key, + # ``ep.api_base`` already carries the section's flat address + # when the endpoint named none of its own (see + # ``provider_endpoints``); the fallback here is only for a + # gateway/local provider whose *flat* address is also empty, + # where ``get_api_base`` still has the spec's default to + # offer. api_base=ep.api_base or config.get_api_base(model), default_model=model, - extra_headers=ep.extra_headers or (p.extra_headers if p else None), + extra_headers=ep.extra_headers, provider_name=provider_name, extra_body=wire_overrides(provider_name, model) or None, model_overrides=config.agents.defaults.model_overrides, @@ -171,9 +177,12 @@ def make_inner(ep): extra_body = wire_overrides(provider_name, model) or None provider = LiteLLMProvider( api_key=eps[0].api_key, + # Same fallback as ``make_inner`` above: only reached when the + # flat address is empty too, for a gateway/local provider's + # spec default. api_base=eps[0].api_base or config.get_api_base(model), default_model=model, - extra_headers=eps[0].extra_headers or (p.extra_headers if p else None), + extra_headers=eps[0].extra_headers, provider_name=provider_name, extra_body=extra_body, model_overrides=config.agents.defaults.model_overrides, diff --git a/raven/config/schema.py b/raven/config/schema.py index 778de815..c4273b08 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -367,8 +367,9 @@ class ProviderConfig(Base): # whose auth is OAuth, or that needs more than a key and an address (Azure # OpenAI, Codex), gets this rejected at `make_provider` construction time # (wired in a later stage; this field exists regardless). Set and non-empty, - # it replaces the flat `api_key`/`api_base` outright rather than merging - # with them -- see `raven.providers.endpoints.provider_endpoints` for the + # it replaces the flat `api_key` outright rather than merging with it; an + # entry naming neither its own `api_base` nor `extra_headers` inherits the + # flat ones -- see `raven.providers.endpoints.provider_endpoints` for the # one place that resolves which of the two shapes (or Gemini's # `api_key_list`) is in effect. endpoints: list[ProviderEndpoint] = Field(default_factory=list) diff --git a/raven/providers/auth.py b/raven/providers/auth.py index 28c5f471..6cd04b70 100644 --- a/raven/providers/auth.py +++ b/raven/providers/auth.py @@ -27,6 +27,8 @@ from pathlib import Path from typing import TYPE_CHECKING, Any +from raven.providers.endpoints import provider_endpoints + if TYPE_CHECKING: from raven.providers.registry import ProviderSpec @@ -131,13 +133,15 @@ def _present(section: Any, name: str) -> bool: Sections reach here as both: the schema object on the routing path, a raw mapping on the display path. - Mirrors the precedence ``provider_endpoints`` reads by: ``endpoints`` set - means the flat fields are ignored outright, not merged with them, so a flat - key alongside a keyless endpoint must not count as present -- that flat key - is never the one a request actually sends. Only when ``endpoints`` is empty - does the flat field (and, for a list field like ``api_key_list``, any - element of it) decide the answer. The gate and the reader must agree on - which shape is in effect; see the ``endpoints`` module docstring. + Consumes ``provider_endpoints`` rather than re-deriving its precedence: + ``endpoints`` set means only the resolved list counts -- flat fields + included, api_key never inherited -- so a flat key alongside a keyless + endpoint must not count as present, and an endpoint missing only its own + address still counts once the section's flat address fills it in. Only + ``api_key_list`` has no counterpart on ``ResolvedEndpoint`` (it collapses + into several per-key entries there), so it is read off the section + directly, and is unsatisfiable once ``endpoints`` is set -- ignored + outright, same as the flat key. """ if section is None: return False @@ -146,7 +150,9 @@ def _present(section: Any, name: str) -> bool: # arbitrary duck-typed objects (test doubles included), and only a real # list is the endpoints shape provider_endpoints reads. if isinstance(endpoints, (list, tuple)) and endpoints: - return any(_present(endpoint, name) for endpoint in endpoints) + if name not in ("api_key", "api_base"): + return False + return any(bool(getattr(ep, name, None)) for ep in provider_endpoints(section)) value = section.get(name) if isinstance(section, dict) else getattr(section, name, None) if isinstance(value, (list, tuple)): return any(bool(v) for v in value) diff --git a/raven/providers/endpoints.py b/raven/providers/endpoints.py index e7718120..629d87c7 100644 --- a/raven/providers/endpoints.py +++ b/raven/providers/endpoints.py @@ -13,12 +13,15 @@ uniform list, so "every endpoint this section offers" is asked once rather than re-derived at each call site with its own idea of the precedence. -The three do not mix. ``endpoints`` set means the flat fields and -``api_key_list`` are both ignored outright, not merged with the list -- a -partial merge is how a stale flat key would outlive the endpoint meant to -replace it. ``api_key_list`` without ``endpoints`` still shares the flat -``api_base``/``extra_headers``: those were never plural, so there is nothing -to choose between for them. +The three do not mix in one respect: ``endpoints`` set means the flat +``api_key`` and ``api_key_list`` are ignored outright, not merged with the +list -- a partial merge of the key is how a stale flat one would outlive the +endpoint meant to replace it. ``api_base``/``extra_headers`` are different: +an entry that names neither inherits the section's flat value, the same way +every ``api_key_list`` entry already shares the flat address -- an +``endpoint add`` that only ever set ``--label``/``--api-key`` is otherwise +unable to run at all, address included, while the very config it wrote passes +every other check. An unconfigured section (no endpoints, no list, no flat key) resolves to one endpoint holding the empty flat values rather than an empty list. That is what @@ -28,19 +31,15 @@ The gate and the reader must answer the same question the same way: ``raven.providers.auth._present``, which decides whether a section is usable -at all, has to mirror this precedence exactly -- ``endpoints`` non-empty means -only the endpoints count, flat fields included, or a section with a healthy -flat key and a keyless endpoint would pass the gate while this function hands -the empty key to every actual request. +at all, calls into this module rather than re-deriving the precedence -- a +section with a healthy flat key and a keyless endpoint must fail the gate +exactly as this function hands the empty key to every actual request. """ from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from raven.config.schema import ProviderConfig +from typing import Any @dataclass(frozen=True) @@ -53,27 +52,41 @@ class ResolvedEndpoint: extra_headers: dict[str, str] | None -def provider_endpoints(section: "ProviderConfig") -> list[ResolvedEndpoint]: +def _field(section: Any, name: str) -> Any: + """Read one field off a section, whether it is a schema object or a raw + mapping -- see ``raven.providers.auth._present`` for why both reach here.""" + return section.get(name) if isinstance(section, dict) else getattr(section, name, None) + + +def provider_endpoints(section: Any) -> list[ResolvedEndpoint]: """Every endpoint ``section`` offers, in the shape it was declared.""" - if section.endpoints: + endpoints = _field(section, "endpoints") + if endpoints: + flat_base = _field(section, "api_base") + flat_headers = _field(section, "extra_headers") return [ ResolvedEndpoint( - label=endpoint.label, - api_key=endpoint.api_key, - api_base=endpoint.api_base, - extra_headers=endpoint.extra_headers, + label=_field(endpoint, "label"), + api_key=_field(endpoint, "api_key") or "", + # An entry that names neither inherits the section's flat + # value -- never the key, which must come from the entry + # itself or not at all (see the module docstring). + api_base=_field(endpoint, "api_base") or flat_base, + extra_headers=_field(endpoint, "extra_headers") or flat_headers, ) - for endpoint in section.endpoints + for endpoint in endpoints ] - key_list = getattr(section, "api_key_list", None) + key_list = _field(section, "api_key_list") if key_list: + flat_base = _field(section, "api_base") + flat_headers = _field(section, "extra_headers") return [ ResolvedEndpoint( label=f"key-{i}", api_key=key, - api_base=section.api_base, - extra_headers=section.extra_headers, + api_base=flat_base, + extra_headers=flat_headers, ) for i, key in enumerate(key_list, start=1) ] @@ -81,8 +94,8 @@ def provider_endpoints(section: "ProviderConfig") -> list[ResolvedEndpoint]: return [ ResolvedEndpoint( label="default", - api_key=section.api_key, - api_base=section.api_base, - extra_headers=section.extra_headers, + api_key=_field(section, "api_key") or "", + api_base=_field(section, "api_base"), + extra_headers=_field(section, "extra_headers"), ) ] diff --git a/tests/test_provider_auth_method.py b/tests/test_provider_auth_method.py index 0ccbf9c0..53fe6243 100644 --- a/tests/test_provider_auth_method.py +++ b/tests/test_provider_auth_method.py @@ -102,6 +102,18 @@ # request -- the gate must say the same, not fall back to the flat key. "section": {"apiKey": "sk-ant-TEST", "endpoints": [{"label": "primary", "apiKey": ""}]}, }, + "hosted_vllm_endpoints_inherit_flat_base": { + "provider": "hosted_vllm", + "model": "hosted_vllm/some-model", + # `endpoint add` without `--api-base` is the common case: the address + # lives on the section, not repeated on every entry. The gate must read + # the same inherited address `provider_endpoints` resolves, not each + # endpoint's own (empty) field. + "section": { + "apiBase": "http://10.0.0.5:8000/v1", + "endpoints": [{"label": "a", "apiKey": "k1"}, {"label": "b", "apiKey": "k2"}], + }, + }, } @@ -251,6 +263,27 @@ def test_flat_key_does_not_paper_over_a_keyless_endpoint(tmp_path: Path) -> None assert not _startup_says(case, path) +def test_endpoints_without_their_own_base_inherit_the_flat_one_at_the_gate() -> None: + """`endpoint add` without `--api-base` must not be refused at startup -- + the gate reads the same inherited address the reader resolves, not each + endpoint's own (empty) field.""" + from raven.config.schema import ProviderConfig + from raven.providers.auth import credential_status + from raven.providers.endpoints import provider_endpoints + + section = ProviderConfig.model_validate( + { + "apiBase": "http://10.0.0.5:8000/v1", + "endpoints": [{"label": "a", "apiKey": "k1"}, {"label": "b", "apiKey": "k2"}], + } + ) + assert credential_status("hosted_vllm", section).ok + assert [e.api_base for e in provider_endpoints(section)] == [ + "http://10.0.0.5:8000/v1", + "http://10.0.0.5:8000/v1", + ] + + def test_credential_status_false_for_flat_key_and_keyless_endpoint() -> None: from raven.config.schema import ProviderConfig from raven.providers.auth import credential_status diff --git a/tests/test_provider_endpoints.py b/tests/test_provider_endpoints.py index 9e3510d0..8c63e4b0 100644 --- a/tests/test_provider_endpoints.py +++ b/tests/test_provider_endpoints.py @@ -42,17 +42,72 @@ def test_endpoints_list_is_used_verbatim() -> None: ] -def test_endpoints_list_takes_priority_over_flat_fields_not_merged() -> None: +def test_endpoints_list_takes_priority_over_flat_key_but_inherits_missing_base_and_headers() -> None: + """``api_key`` never inherits -- a stale flat key must not outlive the + endpoint meant to replace it. ``api_base``/``extra_headers`` do, the same + way an ``api_key_list`` entry already shares the flat address: an + ``endpoint add`` that only ever set ``--label``/``--api-key`` still needs + somewhere to send the request. + """ section = ProviderConfig( api_key="sk-flat", api_base="https://flat.example", extra_headers={"X-Flat": "1"}, + endpoints=[ProviderEndpoint(label="only", api_key="sk-1")], + ) + + resolved = provider_endpoints(section) + + assert resolved == [ + ResolvedEndpoint(label="only", api_key="sk-1", api_base="https://flat.example", extra_headers={"X-Flat": "1"}) + ] + + +def test_endpoint_with_no_key_does_not_inherit_the_flat_key() -> None: + """The one field that never inherits, even though the others do.""" + section = ProviderConfig( + api_key="sk-flat", + api_base="https://flat.example", endpoints=[ProviderEndpoint(label="only")], ) resolved = provider_endpoints(section) - assert resolved == [ResolvedEndpoint(label="only", api_key="", api_base=None, extra_headers=None)] + assert resolved == [ResolvedEndpoint(label="only", api_key="", api_base="https://flat.example", extra_headers=None)] + + +def test_endpoints_own_base_and_headers_win_over_the_flat_ones() -> None: + """Inheritance only fills a gap; an endpoint that names its own wins.""" + section = ProviderConfig( + api_base="https://flat.example", + extra_headers={"X-Flat": "1"}, + endpoints=[ + ProviderEndpoint(label="only", api_key="sk-1", api_base="https://own.example", extra_headers={"X-Own": "2"}) + ], + ) + + resolved = provider_endpoints(section) + + assert resolved == [ + ResolvedEndpoint(label="only", api_key="sk-1", api_base="https://own.example", extra_headers={"X-Own": "2"}) + ] + + +def test_endpoints_without_their_own_base_all_inherit_the_flat_one() -> None: + """``endpoint add`` with no ``--api-base`` must still be able to run -- + every entry missing one falls back to the section's flat address, not just + the first.""" + section = ProviderConfig( + api_base="http://10.0.0.5:8000/v1", + endpoints=[ + ProviderEndpoint(label="a", api_key="k1"), + ProviderEndpoint(label="b", api_key="k2"), + ], + ) + + resolved = provider_endpoints(section) + + assert [e.api_base for e in resolved] == ["http://10.0.0.5:8000/v1", "http://10.0.0.5:8000/v1"] def test_gemini_api_key_list_yields_one_endpoint_per_key() -> None: From c3c7baae947990f78cebaf291e47a0a88469efab Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 20:51:16 +0800 Subject: [PATCH 51/78] fix(providers): a spec's shipped default address satisfies the credential gate `custom` ships a default_api_base, and on main a bare api_key was a runnable config; the redesigned gate required a config-carried address and refused to start it. The address requirement used by requires_api_base specs now falls back to the spec's own default, while azure (no default) still demands one and is_local specs keep the plain requirement -- a local deployment's standard port must not make an untouched section look configured. The picker's save_key handler carried its own copy of that address rule and would have kept refusing what the gate now runs; it consults credential_status instead and names the missing requirement it reports. Co-authored-by: Claude (claude-fable-5) --- raven/providers/auth.py | 40 +++++++++++++++++++++++++----- raven/tui_rpc/methods/model.py | 20 +++++++-------- tests/test_provider_auth_method.py | 14 +++++++++++ tests/test_tui_rpc_model.py | 20 +++++++++++---- 4 files changed, 72 insertions(+), 22 deletions(-) diff --git a/raven/providers/auth.py b/raven/providers/auth.py index 6cd04b70..b1990c4b 100644 --- a/raven/providers/auth.py +++ b/raven/providers/auth.py @@ -53,9 +53,16 @@ class Requirement: fields: tuple[str, ...] label: str hint: str = "" + #: A ``ProviderSpec`` attribute that also satisfies this requirement when + #: truthy, even though the config carries nothing for it -- e.g. custom's + #: ``default_api_base``, a working address the user may still override. + #: Empty for every requirement but ``_ADDRESS``. + spec_fallback: str = "" - def satisfied_by(self, section: Any) -> bool: - return any(_present(section, name) for name in self.fields) + def satisfied_by(self, section: Any, spec: "ProviderSpec | None" = None) -> bool: + if any(_present(section, name) for name in self.fields): + return True + return bool(self.spec_fallback and spec is not None and getattr(spec, self.spec_fallback, None)) @dataclass(frozen=True) @@ -68,7 +75,14 @@ class AuthMethod: checks_token_file: bool = False label: str = "" - def missing(self, section: Any, provider: str, *, include_external: bool) -> list[Requirement]: + def missing( + self, + section: Any, + provider: str, + *, + spec: "ProviderSpec | None" = None, + include_external: bool, + ) -> list[Requirement]: """What this method still needs. ``include_external`` decides whether material held outside the config is @@ -83,7 +97,7 @@ def missing(self, section: Any, provider: str, *, include_external: bool) -> lis if not include_external: return [] return [] if _token_present(provider) else [_SIGN_IN(provider)] - return [req for req in self.requires if not req.satisfied_by(section)] + return [req for req in self.requires if not req.satisfied_by(section, spec)] class MissingCredentialsError(Exception): @@ -210,6 +224,20 @@ def _SIGN_IN(provider: str) -> Requirement: # noqa: N802 - a constructor, named "an API key -- run `raven provider set {public} --api-key ` (or --api-key-list k1,k2)", ) _ADDRESS = Requirement(("api_base",), "an address", "an address -- run `raven provider set {public} --api-base `") +#: Same requirement, plus the spec's own working default. Only for +#: `requires_api_base`: that flag means the *user's* address is mandatory +#: (Azure, a bespoke endpoint) with no config-independent fallback of its own +#: to fall back to -- unless the spec ships one anyway (`custom`'s +#: localhost gateway). `is_local` keeps the plain `_ADDRESS`: a local +#: deployment's spec default (Ollama's standard port) must not make it look +#: configured before the user has pointed it anywhere, which is the bug +#: `_has_credentials`'s docstring already names. +_ADDRESS_OR_SPEC_DEFAULT = Requirement( + ("api_base",), + "an address", + "an address -- run `raven provider set {public} --api-base `", + spec_fallback="default_api_base", +) #: Declarations for the providers whose shape the spec flags cannot express. @@ -242,7 +270,7 @@ def auth_methods(spec: "ProviderSpec | None", name: str = "") -> tuple[AuthMetho if spec.is_local: return (AuthMethod(KIND_NONE, (_ADDRESS,), label="address"),) if spec.requires_api_base: - return (AuthMethod(KIND_API_KEY, (_KEY, _ADDRESS), label="key and endpoint"),) + return (AuthMethod(KIND_API_KEY, (_KEY, _ADDRESS_OR_SPEC_DEFAULT), label="key and endpoint"),) return (AuthMethod(KIND_API_KEY, (_KEY,), label="API key"),) @@ -272,7 +300,7 @@ def credential_status( unsatisfied: list[tuple[AuthMethod, tuple[Requirement, ...]]] = [] for method in methods: - gap = method.missing(section, name, include_external=include_external) + gap = method.missing(section, name, spec=spec, include_external=include_external) if not gap: return CredentialStatus(name, True, method.kind, (), method) unsatisfied.append((method, tuple(_localize(req, name) for req in gap))) diff --git a/raven/tui_rpc/methods/model.py b/raven/tui_rpc/methods/model.py index faad42a1..ec2d276f 100644 --- a/raven/tui_rpc/methods/model.py +++ b/raven/tui_rpc/methods/model.py @@ -277,14 +277,6 @@ async def model_save_key(params: dict) -> dict: data={"slug": parsed.slug}, ) kind = credential_kind(parsed.slug) - # The shape drives which fields to ask for; whether the submission is - # complete is `providers.auth`, the same answer every other gate uses. This - # branch chain was the sixth place deciding that independently. - if kind in (CRED_ENDPOINT, CRED_LOCAL) and not parsed.api_base: - raise ConfigValidationError( - f"{label} requires an api_base", - data={"slug": parsed.slug, "field": "api_base"}, - ) if kind == CRED_LOCAL and parsed.api_key: # Said out loud rather than dropped: a local deployment writes no key, so # storing one silently would look like it had been accepted. @@ -292,11 +284,17 @@ async def model_save_key(params: dict) -> dict: f"{label} is a local deployment and takes no api_key; send api_base instead", data={"slug": parsed.slug, "field": "api_key"}, ) + # Whether the submission is complete is `providers.auth`'s answer, the same + # one every other gate uses -- including which requirement a spec default + # already covers (custom's shipped address). An address rule of this + # handler's own is how the picker refused a submission the gate runs. submitted = {"api_key": parsed.api_key, "api_base": parsed.api_base} - if not credential_status(parsed.slug, submitted).ok and kind != CRED_LOCAL: + status = credential_status(parsed.slug, submitted) + if not status.ok: + req = next(iter(status.missing), None) raise ConfigValidationError( - f"{label} requires an api_key", - data={"slug": parsed.slug, "field": "api_key"}, + f"{label} requires {req.label}" if req else f"{label} is missing credentials", + data={"slug": parsed.slug, "field": req.fields[0] if req else "api_key"}, ) # A local deployment is reached by address and has no key, said explicitly diff --git a/tests/test_provider_auth_method.py b/tests/test_provider_auth_method.py index 53fe6243..86ac7502 100644 --- a/tests/test_provider_auth_method.py +++ b/tests/test_provider_auth_method.py @@ -284,6 +284,20 @@ def test_endpoints_without_their_own_base_inherit_the_flat_one_at_the_gate() -> ] +def test_a_spec_shipped_default_address_satisfies_the_address_requirement() -> None: + """`custom` ships a `default_api_base`, so a bare key is a runnable config + and the gate must say so; `azure_openai` ships none, so its address stays + mandatory. `is_local` specs keep the plain requirement either way: their + default (Ollama's standard port) must not make an untouched section look + configured.""" + from raven.config.schema import ProviderConfig + from raven.providers.auth import credential_status + + assert credential_status("custom", ProviderConfig(api_key="sk-local")).ok + assert not credential_status("azure_openai", ProviderConfig(api_key="sk-azure")).ok + assert not credential_status("ollama_chat", ProviderConfig()).ok + + def test_credential_status_false_for_flat_key_and_keyless_endpoint() -> None: from raven.config.schema import ProviderConfig from raven.providers.auth import credential_status diff --git a/tests/test_tui_rpc_model.py b/tests/test_tui_rpc_model.py index 4c080b0c..20d2ab23 100644 --- a/tests/test_tui_rpc_model.py +++ b/tests/test_tui_rpc_model.py @@ -399,9 +399,19 @@ async def test_options_accepts_session_id(fake_home: Path) -> None: assert "providers" in result -async def test_save_key_custom_without_api_base_rejected(fake_home: Path) -> None: - with pytest.raises(ConfigValidationError): - await model_save_key({"slug": "custom", "api_key": "x"}) +async def test_save_key_custom_key_only_accepted(fake_home: Path) -> None: + """The spec ships a default address, so a bare key is a runnable submission -- + the same answer `credential_status` gives; the picker must not refuse what + `raven provider set custom --api-key` accepts.""" + result = await model_save_key({"slug": "custom", "api_key": "x"}) + assert result["provider"]["authenticated"] is True + + +async def test_save_key_azure_key_only_still_rejected(fake_home: Path) -> None: + """No spec default to fall back on: the address stays mandatory.""" + with pytest.raises(ConfigValidationError) as excinfo: + await model_save_key({"slug": "azure_openai", "api_key": "x"}) + assert excinfo.value.data["field"] == "api_base" # ---------------------------------------------------------------------------- @@ -647,14 +657,14 @@ async def test_save_key_still_requires_a_key_for_a_keyed_provider(fake_home: Pat """Relaxing the field for local deployments must not relax it for the rest.""" with pytest.raises(ConfigValidationError) as excinfo: await model_save_key({"slug": "deepseek", "api_key": ""}) - assert "api_key" in str(excinfo.value) + assert excinfo.value.data["field"] == "api_key" async def test_save_key_requires_an_address_for_a_local_deployment(fake_home: Path) -> None: """Neither field given is not a configured provider.""" with pytest.raises(ConfigValidationError) as excinfo: await model_save_key({"slug": "ollama_chat"}) - assert "api_base" in str(excinfo.value) + assert excinfo.value.data["field"] == "api_base" async def test_options_lists_the_codex_models_the_account_reports( From 4cbb9f02e567dbfa42e3e4c5076c045489e0ccfe Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 20:53:03 +0800 Subject: [PATCH 52/78] fix(config): list_providers mirrors the endpoint precedence The key column checked the flat key first, so a section whose requests are decided by its endpoints list displayed "****set****" off a stale flat key while `configured` -- decided off the endpoints, same as the gate -- said False in the same row; the "(N endpoints)" branch was unreachable whenever a flat key lingered. The endpoints branch now runs first, mirroring the reader's precedence (endpoints > api_key_list > flat). Co-authored-by: Claude (claude-fable-5) --- raven/config/update_providers.py | 12 ++++++++-- tests/test_config_update_providers.py | 34 ++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/raven/config/update_providers.py b/raven/config/update_providers.py index 9781b3ae..b9d325c5 100644 --- a/raven/config/update_providers.py +++ b/raven/config/update_providers.py @@ -596,10 +596,18 @@ def list_providers(*, config_path: Path | None = None) -> list[dict[str, Any]]: api_key_redacted = "OAuth token" if configured else "(empty)" elif is_local: api_key_redacted = "(not needed for local)" if not api_key else "****set****" + # Mirrors the reader's own precedence (endpoints > api_key_list > flat, + # see `provider_endpoints`): a stale flat key left behind by an + # `endpoints` migration must not display as "****set****" while + # `configured` -- decided the same way `credential_status` decides it, + # off the endpoints list -- says otherwise. Checked before the flat/list + # branch below, which used to run first and made this branch, and the + # "(N endpoints)" it reports, unreachable whenever a flat key lingered. + elif endpoints: + suffix = f"({len(endpoints)} endpoints)" + api_key_redacted = f"****set**** {suffix}" if any(ep.api_key for ep in endpoints) else f"(empty) {suffix}" elif api_key or api_key_list: api_key_redacted = "****set****" - elif endpoints and any(ep.api_key for ep in endpoints): - api_key_redacted = f"****set**** ({len(endpoints)} endpoints)" else: api_key_redacted = "(empty)" diff --git a/tests/test_config_update_providers.py b/tests/test_config_update_providers.py index 0c6d9cc8..e28679c1 100644 --- a/tests/test_config_update_providers.py +++ b/tests/test_config_update_providers.py @@ -407,12 +407,40 @@ def test_list_reports_endpoints_only_provider_key_state_consistently(cfg_path: P def test_list_does_not_call_keyless_endpoints_set(cfg_path: Path) -> None: """The mirror direction of the consistency rule: endpoints whose keys are all empty hold no credential, so the key column must not say set while - credential_status says the section is unconfigured.""" - add_provider_endpoint("openrouter", label="a", api_base="https://a.example/v1", config_path=cfg_path) + credential_status says the section is unconfigured. + + Blanked out by hand after the write rather than passed to + ``add_provider_endpoint`` directly: that function now refuses to persist a + keyless endpoint for a key-based provider like openrouter, so a section + shaped like this can only exist from a config written before that rule, or + hand-edited -- ``list_providers`` still has to describe it accurately. + """ + add_provider_endpoint("openrouter", label="a", api_key="k1", api_base="https://a.example/v1", config_path=cfg_path) + data = json.loads(cfg_path.read_text()) + data["providers"]["openrouter"]["endpoints"][0]["apiKey"] = "" + cfg_path.write_text(json.dumps(data)) + + row = {p["name"]: p for p in list_providers(config_path=cfg_path)}["openrouter"] + + assert row["configured"] is False + assert row["api_key_redacted"] == "(empty) (1 endpoints)" + + +def test_list_flat_key_residue_does_not_paper_over_keyless_endpoints(cfg_path: Path) -> None: + """A stale flat ``api_key`` left behind by an ``endpoints`` migration must + not display as set while ``configured`` -- decided off the endpoints list, + same as every other gate -- says the section is not usable. + """ + add_provider_endpoint("openrouter", label="a", api_key="k1", api_base="https://a.example/v1", config_path=cfg_path) + data = json.loads(cfg_path.read_text()) + data["providers"]["openrouter"]["apiKey"] = "stale-flat-key" + data["providers"]["openrouter"]["endpoints"][0]["apiKey"] = "" + cfg_path.write_text(json.dumps(data)) row = {p["name"]: p for p in list_providers(config_path=cfg_path)}["openrouter"] - assert row["api_key_redacted"] == "(empty)" + assert row["configured"] is False + assert row["api_key_redacted"] == "(empty) (1 endpoints)" def test_endpoint_ops_refuse_an_invalid_section_instead_of_wiping_it(cfg_path: Path) -> None: From 47e7196c038c07b916794096a6b3bca9fbf26f89 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 20:54:46 +0800 Subject: [PATCH 53/78] fix(config): endpoint writes require a key for key-credential providers The TUI picker validated only the label, so a blank key persisted a keyless endpoint straight into a key-based provider's rotation, while the CLI demanded a key even for local deployments that legitimately have none -- two write surfaces, two answers. The rule now lives once in the ops layer, derived from the registry's credential kind rather than a vendor list: key-based providers refuse an empty api_key, local deployments keep writing keyless endpoints. The CLI's --api-key becomes optional accordingly, the picker mirrors the check client-side to save a round trip, and ProviderEndpoint.label gains min_length=1 so an all-blank entry cannot be written at all. Co-authored-by: Claude (claude-fable-5) --- raven/cli/provider_commands.py | 4 +- raven/config/schema.py | 2 +- raven/config/update_providers.py | 13 ++++- tests/test_cli_provider_commands.py | 24 +++++++++ tests/test_config_schema.py | 5 ++ tests/test_config_update_providers.py | 28 +++++++++- tests/test_tui_rpc_model.py | 23 ++++++-- ui-tui/src/__tests__/modelPicker.test.tsx | 64 +++++++++++++++++++++++ ui-tui/src/components/modelPicker.tsx | 11 ++++ 9 files changed, 164 insertions(+), 10 deletions(-) diff --git a/raven/cli/provider_commands.py b/raven/cli/provider_commands.py index e999721a..e869cea4 100644 --- a/raven/cli/provider_commands.py +++ b/raven/cli/provider_commands.py @@ -730,7 +730,9 @@ def _parse_extra_headers(value: str) -> dict[str, str] | None: def endpoint_add_cmd( name: str = typer.Argument(..., help="Provider name (e.g. openrouter)"), label: str = typer.Option(..., "--label", help="Idempotency key: an existing label is replaced, not merged"), - api_key: str = typer.Option(..., "--api-key", help="API key for this endpoint"), + api_key: str = typer.Option( + "", "--api-key", help="API key for this endpoint (omit only for a local, keyless deployment)" + ), api_base: str = typer.Option("", "--api-base", help="Base URL for this endpoint"), extra_headers: str = typer.Option("", "--extra-headers", help='Extra headers as JSON, e.g. {"X-Foo": "bar"}'), ): diff --git a/raven/config/schema.py b/raven/config/schema.py index c4273b08..2f699d9f 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -348,7 +348,7 @@ class ProviderEndpoint(Base): edits, so two endpoints in the same list must not share one. """ - label: str + label: str = Field(min_length=1) api_key: str = "" api_base: str | None = None extra_headers: dict[str, str] | None = None diff --git a/raven/config/update_providers.py b/raven/config/update_providers.py index b9d325c5..1bd7dd20 100644 --- a/raven/config/update_providers.py +++ b/raven/config/update_providers.py @@ -32,8 +32,10 @@ from raven.config.schema import ProviderConfig, ProviderEndpoint, ProvidersConfig from raven.providers.endpoints import provider_endpoints from raven.providers.registry import ( + CRED_LOCAL, ProviderSpec, canonical_provider_name, + credential_kind, endpoints_unsupported_reason, find_by_name, names_same_provider, @@ -913,12 +915,21 @@ def add_provider_endpoint( RuntimeError for one that ``endpoints_unsupported_reason`` rejects (Codex, MiniMax OAuth, Azure, or any OAuth section) -- the same rejection ``make_provider`` applies at build time, applied here before the write - rather than left for that later failure to catch. + rather than left for that later failure to catch. Also RuntimeError for an + empty ``api_key`` on a provider whose credential shape needs one -- + derived from the registry (``credential_kind``), not a hardcoded vendor + list, so a local/keyless deployment (``hosted_vllm``, ``ollama_chat``, ...) + keeps writing a keyless endpoint while every key-based provider gets the + same rejection the CLI and the TUI picker both need. """ name = canonical_provider_name(name) reason = endpoints_unsupported_reason(name) if reason: raise RuntimeError(reason) + if not api_key and credential_kind(name) != CRED_LOCAL: + raise RuntimeError( + f"{name} needs an api_key -- only a local, keyless deployment can add an endpoint without one" + ) path = config_path or get_config_path() data = read_raw_or_raise(path) cls, endpoints = _load_provider_endpoints(name, data) diff --git a/tests/test_cli_provider_commands.py b/tests/test_cli_provider_commands.py index 9c6a59c4..4f731d10 100644 --- a/tests/test_cli_provider_commands.py +++ b/tests/test_cli_provider_commands.py @@ -958,6 +958,30 @@ def test_endpoint_add_unknown_provider_exits_1(tmp_config: Path) -> None: assert "Unknown provider" in r.output +def test_endpoint_add_without_a_key_is_refused_for_a_key_based_provider(tmp_config: Path) -> None: + """``--api-key`` is no longer required at the flag level -- the shape-aware + refusal lives in the ops layer, shared with the RPC picker, so this must + still exit non-zero with a readable reason rather than silently persist a + keyless endpoint into the rotation.""" + r = runner.invoke( + app, + ["provider", "endpoint", "add", "openrouter", "--label", "x", "--api-base", "https://a.example/v1"], + ) + assert r.exit_code == 1 + assert "api_key" in r.output + + +def test_endpoint_add_without_a_key_is_allowed_for_a_local_deployment(tmp_config: Path) -> None: + r = runner.invoke( + app, + ["provider", "endpoint", "add", "hosted_vllm", "--label", "x", "--api-base", "http://10.0.0.5:8000/v1"], + ) + assert r.exit_code == 0, r.output + + section = json.loads(tmp_config.read_text(encoding="utf-8"))["providers"]["hosted_vllm"] + assert section["endpoints"][0]["apiKey"] == "" + + def test_endpoint_remove_drops_the_label(tmp_config: Path) -> None: runner.invoke(app, ["provider", "endpoint", "add", "openrouter", "--label", "primary", "--api-key", "k1"]) runner.invoke(app, ["provider", "endpoint", "add", "openrouter", "--label", "backup", "--api-key", "k2"]) diff --git a/tests/test_config_schema.py b/tests/test_config_schema.py index af030ae2..4f5c2ab3 100644 --- a/tests/test_config_schema.py +++ b/tests/test_config_schema.py @@ -86,6 +86,11 @@ def test_endpoint_strategy_rejects_an_unknown_value() -> None: ProviderConfig.model_validate({"endpointStrategy": "random"}) +def test_empty_endpoint_label_is_rejected() -> None: + with pytest.raises(ValidationError): + ProviderEndpoint(label="") + + def test_duplicate_endpoint_labels_are_rejected() -> None: with pytest.raises(ValidationError, match="duplicate endpoint label"): ProviderConfig.model_validate( diff --git a/tests/test_config_update_providers.py b/tests/test_config_update_providers.py index e28679c1..ec6e5e4f 100644 --- a/tests/test_config_update_providers.py +++ b/tests/test_config_update_providers.py @@ -211,9 +211,13 @@ def test_get_endpoints_plaintext_with_redact_false(cfg_path: Path) -> None: def test_get_endpoints_empty_key_renders_as_empty(cfg_path: Path) -> None: - add_provider_endpoint("openrouter", label="a", api_key="", config_path=cfg_path) + # hosted_vllm rather than openrouter: a key-based provider now refuses to + # persist a keyless endpoint (see the write-time tests in the endpoints + # section below) -- a local deployment is the shape that legitimately has + # none, and the redaction rule under test does not depend on which. + add_provider_endpoint("hosted_vllm", label="a", api_base="http://localhost:8000/v1", config_path=cfg_path) - cfg = get_provider_config("openrouter", config_path=cfg_path) + cfg = get_provider_config("hosted_vllm", config_path=cfg_path) assert cfg["endpoints"][0].api_key == "(empty)" @@ -840,6 +844,26 @@ def test_add_provider_endpoint_unknown_provider_raises(cfg_path: Path) -> None: add_provider_endpoint("nonexistent_provider", label="x", api_key="k", config_path=cfg_path) +def test_add_provider_endpoint_refuses_an_empty_key_for_a_key_based_provider(cfg_path: Path) -> None: + """The write-time half of the rule: an endpoint with no key is a request + that will 401, so a key-based provider refuses to persist one -- the same + check the picker and the CLI both need, decided once in the ops layer. + """ + with pytest.raises(RuntimeError, match="api_key"): + add_provider_endpoint("openrouter", label="a", api_base="https://a.example/v1", config_path=cfg_path) + + +def test_add_provider_endpoint_allows_an_empty_key_for_a_local_deployment(cfg_path: Path) -> None: + """Derived from the registry's credential shape (``credential_kind``), not + a hardcoded vendor list: a local deployment has no key to give.""" + endpoints = add_provider_endpoint( + "hosted_vllm", label="a", api_base="http://10.0.0.5:8000/v1", config_path=cfg_path + ) + + assert endpoints[0].api_key == "" + assert endpoints[0].api_base == "http://10.0.0.5:8000/v1" + + @pytest.mark.parametrize("provider", ["azure_openai", "github_copilot"]) def test_add_provider_endpoint_rejects_providers_that_cannot_rotate(provider: str, cfg_path: Path) -> None: """Azure connects through a dedicated client, github_copilot through OAuth -- diff --git a/tests/test_tui_rpc_model.py b/tests/test_tui_rpc_model.py index 20d2ab23..b7931f33 100644 --- a/tests/test_tui_rpc_model.py +++ b/tests/test_tui_rpc_model.py @@ -273,16 +273,29 @@ async def test_add_endpoint_answers_with_the_refreshed_list(fake_home: Path) -> async def test_endpoints_never_hand_back_the_key(fake_home: Path) -> None: """The picker only ever displays this list, and a key it did not need to see - is a key a screenshot can leak.""" - await model_add_endpoint({"slug": "deepseek", "label": "eu", "api_key": "sk-eu-secret"}) - await model_add_endpoint({"slug": "deepseek", "label": "keyless"}) + is a key a screenshot can leak. - by_label = {ep["label"]: ep["api_key"] for ep in (await model_endpoints({"slug": "deepseek"}))["endpoints"]} + ``hosted_vllm`` rather than ``deepseek``: a key-based provider now refuses + a keyless endpoint at write time (see the tests below), so a local + deployment -- which legitimately has none -- is what exercises the + keyless half of this without also asserting the opposite rule. + """ + await model_add_endpoint({"slug": "hosted_vllm", "label": "eu", "api_key": "sk-eu-secret"}) + await model_add_endpoint({"slug": "hosted_vllm", "label": "keyless"}) + + by_label = {ep["label"]: ep["api_key"] for ep in (await model_endpoints({"slug": "hosted_vllm"}))["endpoints"]} assert "sk-eu-secret" not in by_label.values() assert by_label == {"eu": "****set****", "keyless": "(empty)"} +async def test_add_endpoint_without_a_key_is_refused_for_a_key_based_provider(fake_home: Path) -> None: + """Same ops-layer rule the CLI ``endpoint add`` command goes through -- + the picker must not be able to persist what the CLI refuses.""" + with pytest.raises(ConfigValidationError, match="api_key"): + await model_add_endpoint({"slug": "deepseek", "label": "keyless"}) + + async def test_add_endpoint_replaces_the_entry_with_the_same_label(fake_home: Path) -> None: """``label`` is the idempotency key, so re-adding it is how a rotated key is written -- appending a second entry would leave the dead key in rotation.""" @@ -341,7 +354,7 @@ async def test_add_endpoint_rejects_providers_that_cannot_rotate(slug: str, fake async def test_endpoint_handlers_accept_session_id(fake_home: Path) -> None: # The picker passes its session down like it does for every other model.* # call; a strict param model would reject the key otherwise. - await model_add_endpoint({"slug": "deepseek", "label": "eu", "session_id": "tui:default"}) + await model_add_endpoint({"slug": "deepseek", "label": "eu", "api_key": "sk-eu", "session_id": "tui:default"}) await model_endpoints({"slug": "deepseek", "session_id": "tui:default"}) result = await model_remove_endpoint({"slug": "deepseek", "label": "eu", "session_id": "tui:default"}) diff --git a/ui-tui/src/__tests__/modelPicker.test.tsx b/ui-tui/src/__tests__/modelPicker.test.tsx index 7654fa01..117c7239 100644 --- a/ui-tui/src/__tests__/modelPicker.test.tsx +++ b/ui-tui/src/__tests__/modelPicker.test.tsx @@ -713,6 +713,70 @@ describe('ModelPicker', () => { h.unmount() }) + it('refuses to submit an endpoint with no key for a key-based provider', async () => { + // Mirrors the ops-layer rule the RPC/CLI both enforce: an endpoint with no + // key on a key-based provider is a request that will 401, so the picker + // should not even round-trip to the RPC to learn that. + const h = mount([anthropic], method => (method === 'model.endpoints' ? { endpoints: [] } : {})) + await delay(60) + + await h.type(ENTER) + await waitForFrame(h, 'step 2/2') + await h.type('e') + await waitForFrame(h, 'no endpoints') + + // label -> Enter -> (blank key) -> Enter -> (blank base) -> Enter attempts submit. + await h.type('a') + await h.type('eu') + await h.type(ENTER) + await h.type(ENTER) + await h.type(ENTER) + + await waitForFrame(h, 'error: API key is required') + expect(h.gw.request).not.toHaveBeenCalledWith('model.add_endpoint', expect.anything()) + + h.unmount() + }) + + it('allows submitting an endpoint with no key for a local deployment', async () => { + const ollamaConfigured: ModelOptionProvider = { ...ollama, authenticated: true } + const h = mount([ollamaConfigured], (method, params) => { + if (method === 'model.endpoints') { + return { endpoints: [] } + } + + if (method === 'model.add_endpoint') { + return { endpoints: [{ api_base: params.api_base, api_key: '(empty)', label: params.label }] } + } + + return {} + }) + await delay(60) + + await h.type(ENTER) + await waitForFrame(h, 'step 2/2') + await h.type('e') + // Not `'no endpoints'`: the fake terminal's escape stripping mangles that + // exact run at this render depth (see the endpoints-list test above) -- + // `'a adds one'` sits on the same line without falling in the mangled span. + await waitForFrame(h, 'a adds one') + + // label -> Enter -> (blank key) -> Enter -> base -> Enter submits. + await h.type('a') + await h.type('local') + await h.type(ENTER) + await h.type(ENTER) + await h.type('http://10.0.0.5:8000/v1') + await h.type(ENTER) + + expect(h.gw.request).toHaveBeenCalledWith( + 'model.add_endpoint', + expect.objectContaining({ label: 'local', slug: 'ollama_chat' }) + ) + + h.unmount() + }) + it('removes the selected endpoint via model.remove_endpoint', async () => { const h = mount([anthropic], method => { if (method === 'model.endpoints') { diff --git a/ui-tui/src/components/modelPicker.tsx b/ui-tui/src/components/modelPicker.tsx index 75687689..79051fc4 100644 --- a/ui-tui/src/components/modelPicker.tsx +++ b/ui-tui/src/components/modelPicker.tsx @@ -559,6 +559,17 @@ export function ModelPicker({ gw, launcher, onCancel, onSelect, sessionId, suspe const apiKey = endpointInputs.api_key.trim() const apiBase = endpointInputs.api_base.trim() + // Same rule the ops layer enforces on write: only a local, keyless + // deployment may add an endpoint without a key. Checked here too so + // the picker doesn't round-trip to the RPC just to learn that -- the + // RPC error still catches it if this ever runs ahead of stale + // provider metadata. + if (!apiKey && provider.auth_type !== 'local') { + setKeyError('API key is required') + + return + } + setKeySaving(true) setKeyError('') gw.request('model.add_endpoint', { From 262aa2f5a3afb390acd0f1c2cdfd83a018120171 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 20:55:26 +0800 Subject: [PATCH 54/78] fix(config): redact extra_headers values in endpoint display faces extra_headers can carry a secret (an auth header some gateways need alongside the key), and both endpoint display faces handed the values back verbatim. Each value is now masked on its own with the key names left visible -- masking the whole dict as one string would also hide which headers are configured -- in list_provider_endpoints and in the nested-model redaction get_provider_config applies per endpoint. Co-authored-by: Claude (claude-fable-5) --- raven/config/update_providers.py | 26 +++++++++++++++++--- tests/test_config_update_providers.py | 35 +++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/raven/config/update_providers.py b/raven/config/update_providers.py index 1bd7dd20..24ffe166 100644 --- a/raven/config/update_providers.py +++ b/raven/config/update_providers.py @@ -426,6 +426,19 @@ def _redact(value: Any) -> Any: return "****set****" +def _redact_headers(headers: dict[str, str] | None) -> dict[str, str] | None: + """Redact each header's value, keeping the key names visible. + + ``extra_headers`` can carry a secret (an auth header some gateways need + alongside the key) -- masking the whole dict as one ``****set****`` string + would also hide which headers are configured, so each value is redacted on + its own, the same rule every other secret field follows. + """ + if headers is None: + return None + return {key: _redact(value) for key, value in headers.items()} + + def _redact_nested_model(instance: BaseModel) -> BaseModel: """Redact this model's own secret fields, by the same rule as the flat ones. @@ -433,12 +446,18 @@ def _redact_nested_model(instance: BaseModel) -> BaseModel: ``list[BaseModel]`` field like ``ProviderConfig.endpoints`` -- so a caller that walks ``specs`` (field-name keyed) never sees a per-endpoint field and can't redact it. This is applied to each list element instead. + + ``extra_headers`` gets its own rule rather than ``_is_secret_field``'s: it + is a dict of values, not one, and masking the whole thing would also hide + which headers are configured -- see ``_redact_headers``. """ updates = { fname: _redact(getattr(instance, fname)) for fname, finfo in type(instance).model_fields.items() if _is_secret_field(fname, finfo) } + if "extra_headers" in type(instance).model_fields: + updates["extra_headers"] = _redact_headers(getattr(instance, "extra_headers", None)) return instance.model_copy(update=updates) if updates else instance @@ -973,11 +992,12 @@ def remove_provider_endpoint( def list_provider_endpoints(name: str, *, config_path: Path | None = None) -> list[dict[str, Any]]: - """List a provider's ``endpoints``, ``api_key`` redacted for display. + """List a provider's ``endpoints``, secrets redacted for display. Returns one dict per endpoint: ``label``, ``api_key`` (``****set****`` / ``(empty)``, same rule as every other secret field), ``api_base``, - ``extra_headers``. Raises KeyError for an unknown provider. + ``extra_headers`` (values redacted the same way, keys left visible -- see + ``_redact_headers``). Raises KeyError for an unknown provider. """ name = canonical_provider_name(name) path = config_path or get_config_path() @@ -988,7 +1008,7 @@ def list_provider_endpoints(name: str, *, config_path: Path | None = None) -> li "label": ep.label, "api_key": _redact(ep.api_key), "api_base": ep.api_base, - "extra_headers": ep.extra_headers, + "extra_headers": _redact_headers(ep.extra_headers), } for ep in endpoints ] diff --git a/tests/test_config_update_providers.py b/tests/test_config_update_providers.py index ec6e5e4f..ae473873 100644 --- a/tests/test_config_update_providers.py +++ b/tests/test_config_update_providers.py @@ -202,6 +202,21 @@ def test_get_redacts_api_key_nested_inside_endpoints(cfg_path: Path) -> None: assert [ep.label for ep in cfg["endpoints"]] == ["a", "b"] +def test_get_redacts_extra_header_values_keeping_keys_visible(cfg_path: Path) -> None: + add_provider_endpoint( + "openrouter", + label="a", + api_key="k1", + extra_headers={"X-Region": "eu-secret"}, + config_path=cfg_path, + ) + + cfg = get_provider_config("openrouter", config_path=cfg_path) + + assert cfg["endpoints"][0].extra_headers == {"X-Region": "****set****"} + assert "eu-secret" not in repr(cfg) + + def test_get_endpoints_plaintext_with_redact_false(cfg_path: Path) -> None: add_provider_endpoint("openrouter", label="a", api_key="sk-SUPER-SECRET-A", config_path=cfg_path) @@ -925,13 +940,29 @@ def test_list_provider_endpoints_redacts_api_key(cfg_path: Path) -> None: def test_list_provider_endpoints_reports_empty_key(cfg_path: Path) -> None: - add_provider_endpoint("openrouter", label="primary", api_key="", config_path=cfg_path) + # hosted_vllm: a key-based provider (openrouter) now refuses to persist a + # keyless endpoint -- see the write-time tests above. + add_provider_endpoint("hosted_vllm", label="primary", api_base="http://localhost:8000/v1", config_path=cfg_path) - out = list_provider_endpoints("openrouter", config_path=cfg_path) + out = list_provider_endpoints("hosted_vllm", config_path=cfg_path) assert out[0]["api_key"] == "(empty)" +def test_list_provider_endpoints_redacts_extra_header_values(cfg_path: Path) -> None: + add_provider_endpoint( + "openrouter", + label="primary", + api_key="k1", + extra_headers={"X-Region": "eu-secret"}, + config_path=cfg_path, + ) + + out = list_provider_endpoints("openrouter", config_path=cfg_path) + + assert out[0]["extra_headers"] == {"X-Region": "****set****"} + + def test_list_provider_endpoints_default_when_none_configured(cfg_path: Path) -> None: assert list_provider_endpoints("openrouter", config_path=cfg_path) == [] From 156cd342c93a8b394909a3f33187642419af342f Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 20:56:12 +0800 Subject: [PATCH 55/78] perf(tui_rpc): read the config once for all picker rows Every picker row re-parsed the config from disk twice -- once for the list_providers mapping rebuilt inside _build_provider_entry, once for _configured_overlays -- so opening the picker cost two parses per registry provider (22 observed for 21 rows). _entries_off_loop now loads the config once and hands each row its mapping and section; the single-row path keeps loading on its own. A counting test pins the bound at two parses per open. Co-authored-by: Claude (claude-fable-5) --- raven/tui_rpc/methods/model.py | 66 +++++++++++++++++++++++++++------- tests/test_tui_rpc_model.py | 28 +++++++++++++++ 2 files changed, 81 insertions(+), 13 deletions(-) diff --git a/raven/tui_rpc/methods/model.py b/raven/tui_rpc/methods/model.py index ec2d276f..6b72fd91 100644 --- a/raven/tui_rpc/methods/model.py +++ b/raven/tui_rpc/methods/model.py @@ -132,7 +132,12 @@ def _account_models(slug: str, *, configured: bool) -> tuple[str, ...]: return tuple(_stored_spelling(slug, model) for model in account_models()) -def _model_labels(slug: str, models: "list[str]") -> dict[str, dict[str, Any]]: +#: "No section was passed in" marker for the helpers below -- distinct from +#: ``None``, which is what a provider absent from the config resolves to. +_UNLOADED: Any = object() + + +def _model_labels(slug: str, models: "list[str]", *, section: Any = _UNLOADED) -> dict[str, dict[str, Any]]: """Display facts for each offered id, skipping the ones nothing describes. What the user wrote under ``model_overlay`` wins: they are describing their @@ -141,7 +146,7 @@ def _model_labels(slug: str, models: "list[str]") -> dict[str, dict[str, Any]]: """ from raven.providers.catalog import describe - overlays = _configured_overlays(slug) + overlays = _configured_overlays(slug, section=section) out: dict[str, dict[str, Any]] = {} for model in models: row = describe(slug, model, overlay=_overlay_for(overlays, slug, model)) @@ -154,19 +159,25 @@ def _model_labels(slug: str, models: "list[str]") -> dict[str, dict[str, Any]]: return out -def _configured_overlays(slug: str) -> dict[str, Any]: +def _configured_overlays(slug: str, *, section: Any = _UNLOADED) -> dict[str, Any]: """This provider's user-written model descriptions, keyed by merge key. Keyed by identity rather than by the string the user typed, so an overlay written against a bare id still matches the qualified id the picker offers. + + ``section`` lets a caller that already loaded the config hand the + provider's section in (the all-rows path loads once instead of once per + row); left unset, the config is read here. """ - from raven.config.loader import load_config from raven.providers.wire import merge_key - try: - section = load_config().providers.get(slug) - except Exception: - return {} + if section is _UNLOADED: + from raven.config.loader import load_config + + try: + section = load_config().providers.get(slug) + except Exception: + return {} overlay = getattr(section, "model_overlay", None) or {} return {merge_key(slug, model): value for model, value in overlay.items()} @@ -177,9 +188,16 @@ def _overlay_for(overlays: dict[str, Any], slug: str, model: str) -> Any: return overlays.get(merge_key(slug, model)) -def _build_provider_entry(slug: str, *, current_provider: str | None) -> dict[str, Any]: +def _build_provider_entry( + slug: str, + *, + current_provider: str | None, + providers: dict[str, dict[str, Any]] | None = None, + section: Any = _UNLOADED, +) -> dict[str, Any]: spec = find_by_name(slug) - providers = {p["name"]: p for p in list_providers()} + if providers is None: + providers = {p["name"]: p for p in list_providers()} info = providers.get(slug, {}) kind = credential_kind(slug) @@ -195,7 +213,7 @@ def _build_provider_entry(slug: str, *, current_provider: str | None) -> dict[st # model is rather than only what it is called on the wire. Omitted for # ids no catalogue carries -- a local finetune, or a release newer than # the bundled snapshot -- and the picker falls back to the id for those. - "model_labels": _model_labels(slug, models), + "model_labels": _model_labels(slug, models, section=section), "slug": slug, "name": info.get("display_name") or (spec.label if spec else slug), "authenticated": configured, @@ -222,10 +240,32 @@ async def _entry_off_loop(slug: str, current_provider: str | None) -> dict[str, async def _entries_off_loop(current_provider: str | None) -> list[dict[str, Any]]: - """Build every picker row in one thread hop rather than one hop per row.""" + """Build every picker row in one thread hop rather than one hop per row. + + The config is also read once for all rows rather than once per row: + ``_build_provider_entry`` re-derives the ``list_providers`` mapping and + the provider's own section when called for a single row, and both are + hoisted here for the all-rows case. + """ def _build() -> list[dict[str, Any]]: - return [_build_provider_entry(p["name"], current_provider=current_provider) for p in list_providers()] + from raven.config.loader import load_config + + rows = list_providers() + providers = {p["name"]: p for p in rows} + try: + sections = load_config().providers + except Exception: + sections = None + return [ + _build_provider_entry( + p["name"], + current_provider=current_provider, + providers=providers, + section=sections.get(p["name"]) if sections is not None else _UNLOADED, + ) + for p in rows + ] return await asyncio.to_thread(_build) diff --git a/tests/test_tui_rpc_model.py b/tests/test_tui_rpc_model.py index b7931f33..1459a00b 100644 --- a/tests/test_tui_rpc_model.py +++ b/tests/test_tui_rpc_model.py @@ -797,3 +797,31 @@ async def test_a_user_written_overlay_reaches_the_picker(fake_home: Path) -> Non entry = _entry(await model_options({}), "hosted_vllm") label = (entry.get("model_labels") or {}).get("hosted-vllm/my-finetune-v3") assert label == {"label": "Our finetune", "description": "tuned on tickets"} + + +async def test_options_config_reads_do_not_scale_with_the_row_count( + fake_home: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`_entries_off_loop` hoists the config read: one parse for all rows plus + one for the current selection. Without the hoist every row re-parsed the + config from disk (`_configured_overlays`, plus the `list_providers` mapping + in `_build_provider_entry`), so this count sat above the row count instead. + An absolute bound because the row count itself never varies -- the picker + lists every registry provider whether or not it is configured.""" + import raven.config.loader as loader + + _write_config(fake_home, {"providers": {"anthropic": {"api_key": "sk-1"}}}) + real = loader.load_config + calls = 0 + + def counting(*args: object, **kwargs: object): + nonlocal calls + calls += 1 + return real(*args, **kwargs) + + monkeypatch.setattr(loader, "load_config", counting) + result = await model_options({}) + + assert len(result["providers"]) > 2, "too few rows for the bound to mean anything" + assert calls <= 2, f"{calls} config parses for {len(result['providers'])} rows" From 9335b8dadf3aa17d898742be7e05af99225f3be0 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 21:19:45 +0800 Subject: [PATCH 56/78] fix(providers): codex non-200 carries the live status into classification Dropping the 404 text needle left codex's non-200 path -- a plain RuntimeError whose message starts "HTTP 404:" -- classifying as unknown, so a real 404 stopped triggering fallback. The status-carrying exception azure grew for the same reason is promoted to a shared ProviderHTTPError (subclassing RuntimeError, so any broad handler keeps matching) and codex raises it too; both providers now classify from the live status code instead of the rendered text. Co-authored-by: Claude (claude-fable-5) --- raven/providers/azure_openai_provider.py | 20 ++++---------------- raven/providers/base.py | 16 ++++++++++++++++ raven/providers/openai_codex_provider.py | 6 ++++-- tests/test_openai_codex_provider.py | 14 ++++++++++++++ 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/raven/providers/azure_openai_provider.py b/raven/providers/azure_openai_provider.py index 7e419212..be865799 100644 --- a/raven/providers/azure_openai_provider.py +++ b/raven/providers/azure_openai_provider.py @@ -10,25 +10,11 @@ import httpx import json_repair -from raven.providers.base import LLMProvider, LLMResponse, ToolCallRequest +from raven.providers.base import LLMProvider, LLMResponse, ProviderHTTPError, ToolCallRequest _AZURE_MSG_KEYS = frozenset({"role", "content", "tool_calls", "tool_call_id", "name"}) -class _AzureHTTPError(Exception): - """Carries the real status code past the point where it gets rendered into a string. - - ``classify_error`` reads a status code off an exception; a non-200 response - handled here has one (``response.status_code``), but turning it into - ``LLMResponse.content`` loses it unless something exception-shaped carries - it back through, which is what this does. - """ - - def __init__(self, status_code: int, body: str): - super().__init__(f"Azure OpenAI API Error {status_code}: {body}") - self.status_code = status_code - - class AzureOpenAIProvider(LLMProvider): """ Azure OpenAI provider with API version 2024-10-21 compliance. @@ -182,7 +168,9 @@ async def chat( client.post(url, headers=headers, json=payload), self.generation.timeout ) if response.status_code != 200: - exc = _AzureHTTPError(response.status_code, response.text) + exc = ProviderHTTPError( + response.status_code, f"Azure OpenAI API Error {response.status_code}: {response.text}" + ) return LLMResponse( content=str(exc), finish_reason="error", diff --git a/raven/providers/base.py b/raven/providers/base.py index 7e38cd55..b7316e9a 100644 --- a/raven/providers/base.py +++ b/raven/providers/base.py @@ -68,6 +68,22 @@ class ErrorClassification: refuses_prompt_cache: bool = False +class ProviderHTTPError(RuntimeError): + """Carries a real HTTP status past the point where a provider renders its + non-200 response into a string. + + ``classify_error`` reads a status code off a live exception; a provider + that speaks HTTP directly (azure, codex) has one on the response but loses + it the moment the error becomes ``str`` content -- raising or classifying + through this keeps the status attached, instead of regex-guessing it back + out of the rendered text. + """ + + def __init__(self, status_code: int, message: str): + super().__init__(message) + self.status_code = status_code + + @dataclass class ToolCallRequest: """A tool call request from the LLM.""" diff --git a/raven/providers/openai_codex_provider.py b/raven/providers/openai_codex_provider.py index cca3e99c..8c31b338 100644 --- a/raven/providers/openai_codex_provider.py +++ b/raven/providers/openai_codex_provider.py @@ -22,7 +22,7 @@ import httpx from loguru import logger -from raven.providers.base import LLMProvider, LLMResponse, ToolCallRequest +from raven.providers.base import LLMProvider, LLMResponse, ProviderHTTPError, ToolCallRequest DEFAULT_CODEX_URL = "https://chatgpt.com/backend-api/codex/responses" DEFAULT_ORIGINATOR = "raven" @@ -143,7 +143,9 @@ async def _request_codex( async with client.stream("POST", url, headers=headers, json=body) as response: if response.status_code != 200: text = await response.aread() - raise RuntimeError(_friendly_error(response.status_code, text.decode("utf-8", "ignore"))) + raise ProviderHTTPError( + response.status_code, _friendly_error(response.status_code, text.decode("utf-8", "ignore")) + ) return await _consume_sse(response, timeout) diff --git a/tests/test_openai_codex_provider.py b/tests/test_openai_codex_provider.py index 779b3edc..29513cda 100644 --- a/tests/test_openai_codex_provider.py +++ b/tests/test_openai_codex_provider.py @@ -12,6 +12,7 @@ import pytest +from raven.providers.base import ProviderHTTPError from raven.providers.openai_codex_provider import ( DEFAULT_CODEX_URL, OpenAICodexProvider, @@ -19,6 +20,7 @@ _consume_sse, _convert_messages, _convert_tool_output, + _friendly_error, _iter_sse, ) @@ -132,6 +134,18 @@ def test_consume_sse_error_classifies_as_retryable_server_error(): assert classification.should_fallback is True +def test_http_404_classifies_as_model_unavailable_via_the_live_status(): + """The non-200 branch raises ProviderHTTPError so classify_error reads the + real status instead of guessing from the rendered text -- a plain 404 body + carrying none of the model-not-found phrases must still bucket correctly.""" + exc = ProviderHTTPError(404, _friendly_error(404, "Resource not found")) + + classification = OpenAICodexProvider.classify_error(exc) + + assert classification.category == "model_unavailable" + assert classification.should_fallback is True + + _TINY_PNG_URI = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==" From 6704f17eace8f3e84f5ea70c93d7e87d7a569f9f Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 21:20:26 +0800 Subject: [PATCH 57/78] fix(config): endpoint display shows the resolved address, not the raw field list_provider_endpoints (feeding `provider endpoint list` and the TUI picker's endpoint rows) read each entry's own fields, so an endpoint written with only --label/--api-key -- the shape the flat-inheritance change exists for -- displayed an empty address while requests were correctly using the section's flat one, reading as "the config did not take". The display face now consumes provider_endpoints, the same resolved view every request uses. Co-authored-by: Claude (claude-fable-5) --- raven/config/update_providers.py | 13 ++++++++++--- tests/test_config_update_providers.py | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/raven/config/update_providers.py b/raven/config/update_providers.py index 24ffe166..90970071 100644 --- a/raven/config/update_providers.py +++ b/raven/config/update_providers.py @@ -997,12 +997,19 @@ def list_provider_endpoints(name: str, *, config_path: Path | None = None) -> li Returns one dict per endpoint: ``label``, ``api_key`` (``****set****`` / ``(empty)``, same rule as every other secret field), ``api_base``, ``extra_headers`` (values redacted the same way, keys left visible -- see - ``_redact_headers``). Raises KeyError for an unknown provider. + ``_redact_headers``). ``api_base``/``extra_headers`` are the resolved + values each request would actually use -- an entry that names none of its + own shows the section's flat value it inherits (see + ``provider_endpoints``), not an empty field the user reads as "did not + take". Raises KeyError for an unknown provider. """ name = canonical_provider_name(name) path = config_path or get_config_path() data = read_raw_or_raise(path) - _, endpoints = _load_provider_endpoints(name, data) + cls, endpoints = _load_provider_endpoints(name, data) + if not endpoints: + return [] + section = cls.model_validate(_raw_section(data, name)) return [ { "label": ep.label, @@ -1010,7 +1017,7 @@ def list_provider_endpoints(name: str, *, config_path: Path | None = None) -> li "api_base": ep.api_base, "extra_headers": _redact_headers(ep.extra_headers), } - for ep in endpoints + for ep in provider_endpoints(section) ] diff --git a/tests/test_config_update_providers.py b/tests/test_config_update_providers.py index ae473873..b47c20af 100644 --- a/tests/test_config_update_providers.py +++ b/tests/test_config_update_providers.py @@ -963,6 +963,25 @@ def test_list_provider_endpoints_redacts_extra_header_values(cfg_path: Path) -> assert out[0]["extra_headers"] == {"X-Region": "****set****"} +def test_list_provider_endpoints_shows_the_inherited_flat_address(cfg_path: Path) -> None: + """An entry written with only ``--label``/``--api-key`` runs against the + section's flat address (see ``provider_endpoints``); the display face must + show that resolved address, not an empty field the user reads as "the + config did not take".""" + from raven.config.update_providers import set_provider_fields + + set_provider_fields( + "openrouter", {"api_key": "flat-key", "api_base": "https://shared.example/v1"}, config_path=cfg_path + ) + add_provider_endpoint("openrouter", label="a", api_key="k1", config_path=cfg_path) + + out = list_provider_endpoints("openrouter", config_path=cfg_path) + + assert out == [ + {"label": "a", "api_key": "****set****", "api_base": "https://shared.example/v1", "extra_headers": None} + ] + + def test_list_provider_endpoints_default_when_none_configured(cfg_path: Path) -> None: assert list_provider_endpoints("openrouter", config_path=cfg_path) == [] From c443b8243601feb5c6e26bc09d36d56b664e972b Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 21:30:13 +0800 Subject: [PATCH 58/78] fix(providers): one source for which spec defaults the reader serves The credential gate's spec-default fallback checked only that default_api_base was non-empty, while Config.get_api_base serves a default only for a gateway or local deployment -- a second derivation of the same rule, which happened to agree for the two requires_api_base specs that exist today and would silently diverge for a spec carrying a default the reader never hands out. Both now read one registry property (usable_default_api_base), and an invariant test walks every spec asserting the gate never passes a bare-key config whose default the reader would then refuse to serve. Co-authored-by: Claude (claude-fable-5) --- raven/config/schema.py | 4 ++-- raven/providers/auth.py | 23 ++++++++++++----------- raven/providers/registry.py | 11 +++++++++++ tests/test_provider_auth_method.py | 17 +++++++++++++++++ 4 files changed, 42 insertions(+), 13 deletions(-) diff --git a/raven/config/schema.py b/raven/config/schema.py index 2f699d9f..7de87fda 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -966,8 +966,8 @@ def get_api_base(self, model: str | None = None) -> str | None: # (like Moonshot) set their base URL via env vars in _setup_env. if name: spec = find_by_name(name) - if spec and (spec.is_gateway or spec.is_local) and spec.default_api_base: - return spec.default_api_base + if spec and spec.usable_default_api_base: + return spec.usable_default_api_base return None @property diff --git a/raven/providers/auth.py b/raven/providers/auth.py index b1990c4b..4a1c238a 100644 --- a/raven/providers/auth.py +++ b/raven/providers/auth.py @@ -55,8 +55,8 @@ class Requirement: hint: str = "" #: A ``ProviderSpec`` attribute that also satisfies this requirement when #: truthy, even though the config carries nothing for it -- e.g. custom's - #: ``default_api_base``, a working address the user may still override. - #: Empty for every requirement but ``_ADDRESS``. + #: shipped localhost address, a working default the user may still + #: override. Empty for every requirement but ``_ADDRESS_OR_SPEC_DEFAULT``. spec_fallback: str = "" def satisfied_by(self, section: Any, spec: "ProviderSpec | None" = None) -> bool: @@ -224,19 +224,20 @@ def _SIGN_IN(provider: str) -> Requirement: # noqa: N802 - a constructor, named "an API key -- run `raven provider set {public} --api-key ` (or --api-key-list k1,k2)", ) _ADDRESS = Requirement(("api_base",), "an address", "an address -- run `raven provider set {public} --api-base `") -#: Same requirement, plus the spec's own working default. Only for -#: `requires_api_base`: that flag means the *user's* address is mandatory -#: (Azure, a bespoke endpoint) with no config-independent fallback of its own -#: to fall back to -- unless the spec ships one anyway (`custom`'s -#: localhost gateway). `is_local` keeps the plain `_ADDRESS`: a local -#: deployment's spec default (Ollama's standard port) must not make it look -#: configured before the user has pointed it anywhere, which is the bug -#: `_has_credentials`'s docstring already names. +#: Same requirement, plus the spec's own working default -- read through +#: `usable_default_api_base`, the same property `Config.get_api_base` serves +#: from, so the gate can never accept a default the reader then refuses to +#: hand out. Only for `requires_api_base`: that flag means the *user's* +#: address is mandatory (Azure, a bespoke endpoint) -- unless the spec ships +#: one anyway (`custom`'s localhost gateway). `is_local` keeps the plain +#: `_ADDRESS`: a local deployment's spec default (Ollama's standard port) +#: must not make it look configured before the user has pointed it anywhere, +#: which is the bug `_has_credentials`'s docstring already names. _ADDRESS_OR_SPEC_DEFAULT = Requirement( ("api_base",), "an address", "an address -- run `raven provider set {public} --api-base `", - spec_fallback="default_api_base", + spec_fallback="usable_default_api_base", ) diff --git a/raven/providers/registry.py b/raven/providers/registry.py index 8a5faa4e..69d9c862 100644 --- a/raven/providers/registry.py +++ b/raven/providers/registry.py @@ -130,6 +130,17 @@ class ProviderSpec: def label(self) -> str: return self.display_name or self.name.title() + @property + def usable_default_api_base(self) -> str: + """The shipped default address ``Config.get_api_base`` would actually serve. + + Non-empty only for a gateway or local deployment -- a direct vendor's + ``default_api_base`` travels via env vars instead and the reader never + hands it out. Stated once so the credential gate cannot accept a + default the reader then refuses to serve (see ``providers.auth``). + """ + return self.default_api_base if (self.is_gateway or self.is_local) else "" + @property def model_prefix(self) -> str: """Route prefix LiteLLM needs on this provider's model ids. diff --git a/tests/test_provider_auth_method.py b/tests/test_provider_auth_method.py index 86ac7502..04205604 100644 --- a/tests/test_provider_auth_method.py +++ b/tests/test_provider_auth_method.py @@ -298,6 +298,23 @@ def test_a_spec_shipped_default_address_satisfies_the_address_requirement() -> N assert not credential_status("ollama_chat", ProviderConfig()).ok +def test_the_gate_never_accepts_a_default_address_the_reader_will_not_serve() -> None: + """Closed loop over every spec: whenever the gate passes a bare-key config + because of a shipped default, `Config.get_api_base` must serve that same + default -- both read `usable_default_api_base`, and this pins that they + keep doing so.""" + from raven.config.schema import Config + from raven.providers.auth import credential_status + from raven.providers.registry import PROVIDERS + + for spec in PROVIDERS: + if not spec.requires_api_base or spec.is_oauth: + continue + cfg = Config.model_validate({"providers": {spec.name: {"apiKey": "sk-x"}}}) + if credential_status(spec.name, cfg.providers.get(spec.name)).ok: + assert cfg.get_api_base(f"{spec.name}/some-model") == spec.usable_default_api_base != "", spec.name + + def test_credential_status_false_for_flat_key_and_keyless_endpoint() -> None: from raven.config.schema import ProviderConfig from raven.providers.auth import credential_status From cad1d380efbac4d6c3958e70ce4f5a161de3fd2a Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 21:32:33 +0800 Subject: [PATCH 59/78] fix(tui_rpc): the picker demands an address only when the gate does needs_api_base was derived from the credential shape alone, so the TUI picker kept blocking a bare-key custom submission client-side -- the exact refusal the save_key alignment was supposed to end -- while rendering the address field as "(required)". It now says "an address must be supplied": true for local deployments (the address is the credential) and for endpoint-credential specs shipping no usable default (azure), false for custom, whose shipped localhost default the gate accepts. The picker's existing check and label read the flag unchanged. Co-authored-by: Claude (claude-fable-5) --- raven/tui_rpc/methods/model.py | 6 +++++- tests/test_cli_onboard_commands.py | 3 ++- tests/test_tui_rpc_model.py | 7 ++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/raven/tui_rpc/methods/model.py b/raven/tui_rpc/methods/model.py index 6b72fd91..4d6d90ac 100644 --- a/raven/tui_rpc/methods/model.py +++ b/raven/tui_rpc/methods/model.py @@ -222,7 +222,11 @@ def _build_provider_entry( "key_env": (spec.env_key or None) if spec else None, "models": models, "total_models": len(models), - "needs_api_base": kind in (CRED_ENDPOINT, CRED_LOCAL), + # "An address must be supplied" -- the gate's answer, not the shape's: + # an endpoint-credential spec that ships a usable default (custom's + # localhost gateway) runs on a bare key, and the picker must not + # demand what the gate does not. + "needs_api_base": kind == CRED_LOCAL or (kind == CRED_ENDPOINT and not (spec and spec.usable_default_api_base)), "warning": warning, } diff --git a/tests/test_cli_onboard_commands.py b/tests/test_cli_onboard_commands.py index 3b1d17fd..24e5f0c4 100644 --- a/tests/test_cli_onboard_commands.py +++ b/tests/test_cli_onboard_commands.py @@ -3680,7 +3680,8 @@ def test_the_model_picker_reports_the_same_credential_shape_as_the_wizard() -> N entry = _build_provider_entry(spec.name, current_provider=None) kind = credential_kind(spec.name) assert entry["auth_type"] == kind, spec.name - assert entry["needs_api_base"] is (kind in (CRED_ENDPOINT, CRED_LOCAL)), spec.name + expected_needs_base = kind == CRED_LOCAL or (kind == CRED_ENDPOINT and not spec.usable_default_api_base) + assert entry["needs_api_base"] is expected_needs_base, spec.name def test_configuring_azure_stores_the_endpoint_it_was_given(tmp_env: Path, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_tui_rpc_model.py b/tests/test_tui_rpc_model.py index 1459a00b..30b1ffc7 100644 --- a/tests/test_tui_rpc_model.py +++ b/tests/test_tui_rpc_model.py @@ -150,10 +150,15 @@ async def test_options_oauth_provider_warning_and_auth_type(fake_home: Path) -> async def test_options_needs_api_base_flag(fake_home: Path) -> None: + """True only when the gate itself demands an address: azure ships no + default, a local deployment's address IS the credential -- while custom + runs on a bare key over its shipped localhost default, so the picker must + not block the submission the gate accepts.""" _write_config(fake_home, {"agents": {"defaults": {"model": "anthropic/claude-sonnet-4-5"}}}) result = await model_options({}) - assert _entry(result, "custom")["needs_api_base"] is True + assert _entry(result, "custom")["needs_api_base"] is False assert _entry(result, "azure_openai")["needs_api_base"] is True + assert _entry(result, "ollama_chat")["needs_api_base"] is True assert _entry(result, "anthropic")["needs_api_base"] is False From 306a05da3dac69db4249232fc57df46a182f183b Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 21:35:17 +0800 Subject: [PATCH 60/78] fix(config): redact the section-level extra_headers values as well The per-endpoint extra_headers got per-value redaction while the flat section field -- where an AiHubMix APP-Code actually lives, per its own comment -- still printed plaintext through `provider get` without --show-secrets: one table, two rules. The field now carries the schema-level secret marker (the direction _is_secret_field's docstring already names) and _redact learned to mask a dict per value, keys left visible. Co-authored-by: Claude (claude-fable-5) --- raven/config/schema.py | 4 +++- raven/config/update_providers.py | 5 ++++- tests/test_config_update_providers.py | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/raven/config/schema.py b/raven/config/schema.py index 7de87fda..6986aa9d 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -359,7 +359,9 @@ class ProviderConfig(Base): api_key: str = "" api_base: str | None = None - extra_headers: dict[str, str] | None = None # Custom headers (e.g. APP-Code for AiHubMix) + # Custom headers (e.g. APP-Code for AiHubMix) -- can carry a secret, so + # display faces redact the values (keys stay visible). + extra_headers: dict[str, str] | None = Field(default=None, json_schema_extra={"secret": True}) models: list[str] = Field(default_factory=list) # User-curated model names for the picker # Several full url/key/header groups under one provider section, for a # vendor reachable by more than one account or region. Meaningful only for diff --git a/raven/config/update_providers.py b/raven/config/update_providers.py index 90970071..2eb83055 100644 --- a/raven/config/update_providers.py +++ b/raven/config/update_providers.py @@ -418,11 +418,14 @@ def _set_nested(dotted_key: str, value: Any, target: dict[str, Any]) -> Any: def _redact(value: Any) -> Any: - """Redact a single value or list of values.""" + """Redact a single value, list of values, or dict of values (per value, + keys left visible -- see ``_redact_headers``).""" if value in (None, "", [], {}): return "(empty)" if isinstance(value, list): return ["****set****" for _ in value] + if isinstance(value, dict): + return _redact_headers(value) return "****set****" diff --git a/tests/test_config_update_providers.py b/tests/test_config_update_providers.py index b47c20af..df67ddaa 100644 --- a/tests/test_config_update_providers.py +++ b/tests/test_config_update_providers.py @@ -202,6 +202,20 @@ def test_get_redacts_api_key_nested_inside_endpoints(cfg_path: Path) -> None: assert [ep.label for ep in cfg["endpoints"]] == ["a", "b"] +def test_get_redacts_flat_extra_header_values_too(cfg_path: Path) -> None: + """The section-level ``extra_headers`` (an AiHubMix APP-Code lives there) + must follow the same per-value rule as the per-endpoint dict -- one table, + one rule.""" + set_provider_fields("aihubmix", {"api_key": "k", "extra_headers": {"APP-Code": "SECRET-VALUE"}}, config_path=cfg_path) + + cfg = get_provider_config("aihubmix", config_path=cfg_path) + + assert cfg["extra_headers"] == {"APP-Code": "****set****"} + assert "SECRET-VALUE" not in repr(cfg) + plain = get_provider_config("aihubmix", redact_secrets=False, config_path=cfg_path) + assert plain["extra_headers"] == {"APP-Code": "SECRET-VALUE"} + + def test_get_redacts_extra_header_values_keeping_keys_visible(cfg_path: Path) -> None: add_provider_endpoint( "openrouter", From 23aef6cde8071056b229a032f0be4e7078d84980 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 21:38:08 +0800 Subject: [PATCH 61/78] fix(tui_rpc): save_key names every missing requirement at once Reporting only the first gap made a bare azure submission a two-round trip: "requires an API key", then "requires an address" after the key was supplied. All missing labels are joined into one message, and the field picked for error.data no longer assumes the first requirement carries one. Co-authored-by: Claude (claude-fable-5) --- raven/tui_rpc/methods/model.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/raven/tui_rpc/methods/model.py b/raven/tui_rpc/methods/model.py index 4d6d90ac..c805ae1e 100644 --- a/raven/tui_rpc/methods/model.py +++ b/raven/tui_rpc/methods/model.py @@ -335,10 +335,11 @@ async def model_save_key(params: dict) -> dict: submitted = {"api_key": parsed.api_key, "api_base": parsed.api_base} status = credential_status(parsed.slug, submitted) if not status.ok: - req = next(iter(status.missing), None) + labels = ", ".join(req.label for req in status.missing) + field = next((f for req in status.missing for f in req.fields), "api_key") raise ConfigValidationError( - f"{label} requires {req.label}" if req else f"{label} is missing credentials", - data={"slug": parsed.slug, "field": req.fields[0] if req else "api_key"}, + f"{label} requires {labels}" if labels else f"{label} is missing credentials", + data={"slug": parsed.slug, "field": field}, ) # A local deployment is reached by address and has no key, said explicitly From f888d86eedaa27fa79aa77f19cb4cbe003321688 Mon Sep 17 00:00:00 2001 From: KT Date: Mon, 10 Aug 2026 21:38:46 +0800 Subject: [PATCH 62/78] fix(providers): the key hint names endpoint add when endpoints exist For a section whose endpoints all lack keys, the gate's hint said `provider set --api-key` -- which writes the flat field the gate then ignores exactly because endpoints exist, so following the hint changed nothing and the gate repeated itself. The hint now names `provider endpoint add` whenever the section carries endpoints. Co-authored-by: Claude (claude-fable-5) --- raven/providers/auth.py | 17 +++++++++++++---- tests/test_provider_auth_method.py | 16 ++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/raven/providers/auth.py b/raven/providers/auth.py index 4a1c238a..08b390b1 100644 --- a/raven/providers/auth.py +++ b/raven/providers/auth.py @@ -304,7 +304,7 @@ def credential_status( gap = method.missing(section, name, spec=spec, include_external=include_external) if not gap: return CredentialStatus(name, True, method.kind, (), method) - unsatisfied.append((method, tuple(_localize(req, name) for req in gap))) + unsatisfied.append((method, tuple(_localize(req, name, section=section) for req in gap))) # Report against the first declared method: it is the preferred one, so its # gap is the shortest path to a working provider. @@ -379,13 +379,22 @@ def key_refusal(vendor: str) -> str | None: return _KEY_CANNOT_CONFIGURE.get(normalize_provider_name(vendor)) -def _localize(req: Requirement, name: str) -> Requirement: +def _localize(req: Requirement, name: str, section: Any = None) -> Requirement: """Put the provider's own name into the hint, so it can be pasted. The hint names a command, not a config path: telling someone to hand-edit `~/.raven/config.json` asks them to know a file layout the CLI exists to hide, and the OAuth hint already gave a command. + + When the section carries ``endpoints``, a key hint must name + ``endpoint add``: the flat field the generic hint writes to is ignored the + moment endpoints exist (see ``_present``), so following that hint changes + nothing and the gate repeats itself. """ - if "{public}" not in req.hint: + hint = req.hint + endpoints = section.get("endpoints") if isinstance(section, dict) else getattr(section, "endpoints", None) + if "api_key" in req.fields and endpoints: + hint = "an API key on an endpoint -- run `raven provider endpoint add {public} --label