From b4cb41feb57618022d6216d6c8a002179a9e404f Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 06:49:01 +0530 Subject: [PATCH 01/38] feat: multi-provider benchmark runners, harness fixes, and tooling Per-provider runners (Fireworks/OpenRouter/native Gemini) and 9 correctness fixes carried over from the benchmarking work: Spider2 query-timeout guard + unclosed-fence SQL extraction, MAX_IN_FLIGHT concurrency caps, OpenRouter input_audio (vs silently-dropped audio_url), OpenRouter client timeout, lowercase API-key fallbacks, and per-model checkpoint/slug tagging. Adds scripts/report_scores.py, BFCL setup, chart/sheet tools, and dev/test deps. Downloaded datasets (spider2_lite/data ~4GB, olmocr full_data) and eval scratch zips excluded via .gitignore. --- .gitignore | 13 + README.md | 113 + benchmarks/asr/_probe_voxpopuli_missing.py | 11 +- benchmarks/asr/voxpopuli_aa_multi.py | 214 +- benchmarks/gpqa/gpqa_fireworks.py | 324 ++ benchmarks/gpqa/gpqa_openai.py | 91 +- benchmarks/gpqa/gpqa_openrouter.py | 4 +- benchmarks/mmmlu/mmmlu_multi.py | 190 +- benchmarks/mmmu_pro/mmmu_pro_multi.py | 216 +- .../obj_detection/reeval_format_tolerant.py | 82 +- benchmarks/obj_detection/refcoco_multi.py | 304 +- .../ocrbench_v2/ocrbench_v2_fireworks.py | 441 ++ benchmarks/ocrbench_v2/ocrbench_v2_gemini.py | 4 + .../olmocr/bench/runners/run_fireworks.py | 99 + benchmarks/olmocr/bench/runners/run_gemini.py | 102 + benchmarks/olmocr/olmocr_bench_fireworks.py | 279 ++ benchmarks/olmocr/olmocr_bench_gemini.py | 280 ++ benchmarks/spider2_lite/spider2_lite.py | 232 +- pyproject.toml | 17 + scripts/bfcl.sh | 47 + scripts/plot_gemini_vs_deepseek.py | 277 ++ scripts/plot_inkling_radar.py | 380 ++ scripts/report_scores.py | 537 +++ scripts/run_fireworks_suite.sh | 108 + scripts/setup_bfcl.sh | 251 + scripts/sheet_columns.py | 209 + src/commons.py | 33 +- src/commons_fireworks.py | 66 + src/commons_openrouter.py | 16 +- uv.lock | 4132 +++++++++++++++++ 30 files changed, 8839 insertions(+), 233 deletions(-) create mode 100644 benchmarks/gpqa/gpqa_fireworks.py create mode 100644 benchmarks/ocrbench_v2/ocrbench_v2_fireworks.py create mode 100644 benchmarks/olmocr/bench/runners/run_fireworks.py create mode 100644 benchmarks/olmocr/bench/runners/run_gemini.py create mode 100644 benchmarks/olmocr/olmocr_bench_fireworks.py create mode 100644 benchmarks/olmocr/olmocr_bench_gemini.py create mode 100755 scripts/bfcl.sh create mode 100644 scripts/plot_gemini_vs_deepseek.py create mode 100644 scripts/plot_inkling_radar.py create mode 100644 scripts/report_scores.py create mode 100755 scripts/run_fireworks_suite.sh create mode 100755 scripts/setup_bfcl.sh create mode 100644 scripts/sheet_columns.py create mode 100644 src/commons_fireworks.py create mode 100644 uv.lock diff --git a/.gitignore b/.gitignore index 3c57a4e..c338fdc 100644 --- a/.gitignore +++ b/.gitignore @@ -208,3 +208,16 @@ __marimo__/ # olmOCR-Bench dataset + per-candidate prediction outputs (large, downloaded from HF) benchmarks/olmocr_bench/olmocr_bench/ + +# Third-party benchmark checkouts + their venvs (see scripts/setup_bfcl.sh) +external/ + +# Downloaded benchmark datasets (large; fetched at runtime, not source) +benchmarks/spider2_lite/data/ +benchmarks/olmocr/bench/full_data/ + +# OCRBench spotting-eval scratch (generated during scoring) +benchmarks/ocrbench_v2/eval_scripts/spotting_eval/gt/ +benchmarks/ocrbench_v2/eval_scripts/spotting_eval/submit/ +benchmarks/ocrbench_v2/eval_scripts/spotting_eval/*.zip +benchmarks/ocrbench_v2/results.zip diff --git a/README.md b/README.md index 736167d..eb8f6b2 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ OPENAI_API_KEY=... ANTHROPIC_API_KEY=... GEMINI_KEY=... OPENROUTER_API_KEY=... +FIREWORKS_API_KEY=... JIGSAWSTACK_API_KEY=... # only for benchmarks/obj_detection/ob_det_api.py ``` @@ -26,6 +27,111 @@ Every runner accepts `--limit N` for a smoke test and `--evaluate-only` / `--predict-only` to split prediction and scoring. Re-running a benchmark resumes from its checkpoint file — already-completed samples are skipped. +Two extra setup steps for specific benchmarks: + +```bash +# olmOCR-bench scoring renders equations in headless chromium +uv run python -m playwright install chromium + +# OCRBench v2 scoring needs NLTK corpora. Run it from OUTSIDE the repo root: +# nltk blocks imports resolving inside the CWD, and .venv lives in the project, +# so `import nltk` from here trips its guard. +(cd /tmp && uv run --project "$OLDPWD" python -c " +import nltk; [nltk.download(p, quiet=True) for p in ('wordnet','omw-1.4','punkt','punkt_tab')]") + +# GPQA's dataset (Idavidrein/gpqa) is gated — accept the terms on the Hub, then +hf auth login +``` + +## BFCL (Berkeley Function Calling Leaderboard) + +Links: [repo](https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard) · [leaderboard](https://gorilla.cs.berkeley.edu/leaderboard.html) + +BFCL lives in its own third-party checkout under `external/` (gitignored) with a +separate venv — its pins conflict with this repo's. `scripts/setup_bfcl.sh` is +the durable artifact: it clones, builds the venv, writes BFCL's `.env`, and +registers the Fireworks models idempotently. Re-run it any time. + +```bash +scripts/setup_bfcl.sh # once; safe to re-run + +# OpenRouter-hosted models (registered by the setup script) +scripts/bfcl.sh generate --model openrouter-qwen3.5-35b-a3b-FC \ + --test-category simple_python --num-threads 4 +scripts/bfcl.sh evaluate --model openrouter-qwen3.5-35b-a3b-FC --test-category simple_python + +scripts/bfcl.sh generate --model openrouter-gemma-4-31b-it-FC \ + --test-category simple_python --num-threads 4 +scripts/bfcl.sh evaluate --model openrouter-gemma-4-31b-it-FC --test-category simple_python +``` + +`evaluate` re-scores from saved responses, so a registration fix (see +`underscore_to_dot` below) costs nothing to apply — no regeneration. + +Use `scripts/bfcl.sh` rather than the `bfcl` CLI directly — it exports +`BFCL_PROJECT_ROOT` (which BFCL needs *before* start, since it derives its +`.env` path from it) and normalises the lowercase `fireworks_api_key` in this +repo's `.env` to the uppercase name BFCL's handler reads. + +Three sharp edges worth knowing: + +- This is BFCL **v4**: the category is `simple_python`, not `simple`. +- A Fireworks run needs a non-empty `OPENAI_API_KEY` even though it never calls + OpenAI — `FireworksHandler.__init__` calls `super().__init__()`, and the parent + `OpenAICompletionsHandler` builds an OpenAI client that raises without one. + The setup script writes a placeholder. +- `soundfile` is an undeclared transitive dependency (`model_config.py` imports + every handler eagerly → `qwen.py` → `qwen_agent` → `soundfile`). Without it, + loading the model registry fails, so *no* model can run. +- **`underscore_to_dot` must match the provider, or correct answers score wrong.** + BFCL's test set uses dotted function names (`math.factorial`), but OpenAI-style + tool schemas forbid `.`, so OpenRouter sanitises them and the model replies + `math_factorial`. With the flag set wrong, every such call fails as + `wrong_func_name`: measured 55.0% vs 93.8% for the same responses. OpenRouter + needs `True`; Fireworks passes dots through and needs `False`. The setup script + encodes this per provider and *refreshes* existing entries on re-run, so a + corrected flag actually takes effect. + +Measured on `simple_python` (400 cases, function-calling mode): + +| Model | Accuracy | +|---|---| +| Qwen3.5-35B-A3B (OpenRouter) | **93.75%** (375/400) | +| Qwen3.7-Plus (Fireworks) | **92.75%** (371/400) | +| Gemma-4-31B-IT (OpenRouter) | **92.25%** (369/400) | + +### Fireworks / Inkling + +`--provider fireworks` (or the `*_fireworks` runners) targets Fireworks-hosted +models over their OpenAI-compatible endpoint. Inkling takes text, image, and +audio input, so it runs on every benchmark here: + +```bash +uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks +uv run -m benchmarks.olmocr.olmocr_bench_fireworks +uv run -m benchmarks.gpqa.gpqa_fireworks +uv run -m benchmarks.mmmlu.mmmlu_multi --provider fireworks --model accounts/fireworks/models/inkling +uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider fireworks --model accounts/fireworks/models/inkling --setting standard +uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider fireworks --model accounts/fireworks/models/inkling --setting vision +uv run -m benchmarks.obj_detection.refcoco_multi --provider fireworks --model accounts/fireworks/models/inkling +uv run -m benchmarks.asr.voxpopuli_aa_multi --provider fireworks --model accounts/fireworks/models/inkling +uv run -m benchmarks.spider2_lite.spider2_lite --provider fireworks +``` + +Two caveats when reading Fireworks numbers: + +- **Reasoning can't be turned off.** Inkling's `reasoning_effort` accepts + `none|low|medium|high|xhigh|max`; `none` is a floor, not a disable, and still + emits reasoning tokens (measured: ~240 on MMMU-Pro standard, ~1100 on + MMMU-Pro vision). Runs tagged `reasoningoff` here are therefore *not* + like-for-like with providers whose thinking is genuinely disabled. Per-sample + `reasoning_tokens` are recorded in the output JSONL so the real spend is + auditable. +- **`inkling-small` is not on serverless.** Only + `accounts/fireworks/models/inkling` is serverless; `inkling-small` needs an + on-demand dedicated deployment first. Once deployed, pass it via `--model` + and every runner writes to its own separate checkpoint. + --- ## OCRBench v2 @@ -44,6 +150,7 @@ uv run -m benchmarks.ocrbench_v2.ocrbench_v2_gemini uv run -m benchmarks.ocrbench_v2.ocrbench_v2_gemini_pro_31 uv run -m benchmarks.ocrbench_v2.ocrbench_v2_grok uv run -m benchmarks.ocrbench_v2.ocrbench_v2_kimi # via OpenRouter +uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks # Inkling # Text-spotting EN subset only uv run -m benchmarks.ocrbench_v2.ocrbench_v2_text_spotting_en @@ -66,6 +173,7 @@ uv run -m benchmarks.olmocr.olmocr_bench uv run -m benchmarks.olmocr.olmocr_bench_openai_mini uv run -m benchmarks.olmocr.olmocr_bench_gemini_pro_31 uv run -m benchmarks.olmocr.olmocr_bench_grok +uv run -m benchmarks.olmocr.olmocr_bench_fireworks # Useful flags uv run -m benchmarks.olmocr.olmocr_bench --sample # tiny sample dataset @@ -183,6 +291,10 @@ uv run -m benchmarks.gpqa.gpqa_gemini --model gemini-3.1-pro-preview uv run -m benchmarks.gpqa.gpqa_openrouter --model x-ai/grok-4.3 --thinking on uv run -m benchmarks.gpqa.gpqa_openrouter --model moonshotai/kimi-k2.6 +# Fireworks (Inkling) +uv run -m benchmarks.gpqa.gpqa_fireworks +uv run -m benchmarks.gpqa.gpqa_fireworks --thinking on --effort high + # Evaluate only uv run -m benchmarks.gpqa.gpqa_openai --evaluate-only ``` @@ -201,6 +313,7 @@ uv run -m benchmarks.spider2_lite.fetch_data # Run uv run -m benchmarks.spider2_lite.spider2_lite +uv run -m benchmarks.spider2_lite.spider2_lite --provider fireworks uv run -m benchmarks.spider2_lite.spider2_lite --predict-only uv run -m benchmarks.spider2_lite.spider2_lite --evaluate-only ``` diff --git a/benchmarks/asr/_probe_voxpopuli_missing.py b/benchmarks/asr/_probe_voxpopuli_missing.py index 8ebfe05..b1d5969 100644 --- a/benchmarks/asr/_probe_voxpopuli_missing.py +++ b/benchmarks/asr/_probe_voxpopuli_missing.py @@ -1,6 +1,3 @@ -"""One-off probe: run the single VoxPopuli-AA sample that all three Gemini runs -dropped, and dump the raw response (text + finish reason + prompt/safety -feedback) so we can see why it was rejected.""" import sys from pathlib import Path @@ -11,7 +8,13 @@ from google import genai from google.genai import types -from benchmarks.asr.voxpopuli_aa import PROMPT, DATASET_ID, SPLIT, build_sample, fetch_audio_bytes +from benchmarks.asr.voxpopuli_aa import ( + DATASET_ID, + PROMPT, + SPLIT, + build_sample, + fetch_audio_bytes, +) from benchmarks.asr.voxpopuli_aa_multi import _load_interfaze_env TARGET_ID = "20150527-0900-PLENARY-14-en_20150527-21:41:31_1" diff --git a/benchmarks/asr/voxpopuli_aa_multi.py b/benchmarks/asr/voxpopuli_aa_multi.py index e04f8a4..1ea2f09 100644 --- a/benchmarks/asr/voxpopuli_aa_multi.py +++ b/benchmarks/asr/voxpopuli_aa_multi.py @@ -13,36 +13,75 @@ uv run -m benchmarks.asr.voxpopuli_aa_multi --provider gemini --model gemini-3-flash-preview --limit 5 """ +import os import sys import json import time +import base64 import asyncio import argparse import traceback from pathlib import Path from datasets import load_dataset +from dotenv import load_dotenv from jiwer import wer, cer from tqdm import tqdm from tqdm.asyncio import tqdm_asyncio +load_dotenv() + +# reasoning.effort sent to OpenRouter-hosted models. Override per run. +OPENROUTER_EFFORT = os.getenv("BENCH_OPENROUTER_EFFORT", "none") + +def _gemini_floor(model: str) -> str: + """Lowest thinking_level the given Gemini model actually accepts. + + gemini-3.7+ flash rejects MINIMAL server-side ("Thinking level MINIMAL is + not supported for this model"), and the SDK enum has no "off"/"none", so + "low" is its true floor. Older 3.x flash models still take "minimal". + BENCH_GEMINI_FLOOR overrides for anything not covered here. + """ + override = os.getenv("BENCH_GEMINI_FLOOR") + if override: + return override + m = model.lower() + if any(v in m for v in ("3.7", "3.8", "3.9", "4.")): + return "low" + return "minimal" + PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent sys.path.insert(0, str(PROJECT_ROOT)) # Reuse everything shared: prompt, WER normalization, JSONL writer, etc. from benchmarks.asr.voxpopuli_aa import ( # noqa: E402 - PROMPT, DATASET_ID, SPLIT, RATE_LIMIT, MAX_RETRIES, - RateLimiter, JsonlWriter, normalize_text, fetch_audio_bytes, - build_sample, load_completed_ids, load_records, - compute_metrics, print_summary, + PROMPT, + DATASET_ID, + SPLIT, + RATE_LIMIT, + MAX_RETRIES, + RateLimiter, + JsonlWriter, + normalize_text, + fetch_audio_bytes, + build_sample, + load_completed_ids, + load_records, + compute_metrics, + print_summary, ) RESULTS_DIR = PROJECT_ROOT / "results" def _load_interfaze_env() -> dict: + """Optional key source — the project .env (loaded above) covers providers + keyed there, so a missing file is not fatal.""" + path = Path.home() / "interfaze" / ".env.local" + if not path.exists(): + return {} env = {} - for line in (Path.home() / "interfaze" / ".env.local").read_text().splitlines(): + for line in path.read_text().splitlines(): line = line.strip() if not line or line.startswith("#") or "=" not in line: continue @@ -53,14 +92,16 @@ def _load_interfaze_env() -> dict: # -------- provider adapters: (audio_bytes, prompt, model, client) -> (content, req_id) -------- + def call_gemini(audio_bytes: bytes, prompt_text: str, model: str, client): """Thinking OFF (or as close as the model allows): - - Gemini 3.x Pro: thinking_level="low" (Pro rejects 'minimal'; min is 'low'). - - Gemini 3.x Flash: thinking_level="minimal" (true disable not supported). - - Gemini 2.5 Pro: thinking_budget=128 (Pro can't go lower than 128). - - Gemini 2.5 Flash: thinking_budget=0 (true disable). - Input audio as inline bytes (Gemini accepts up to ~20 MB inline).""" + - Gemini 3.x Pro: thinking_level="low" (Pro rejects 'minimal'; min is 'low'). + - Gemini 3.x Flash: thinking_level="minimal" (true disable not supported). + - Gemini 2.5 Pro: thinking_budget=128 (Pro can't go lower than 128). + - Gemini 2.5 Flash: thinking_budget=0 (true disable). + Input audio as inline bytes (Gemini accepts up to ~20 MB inline).""" from google.genai import types + m = model.lower() if m.startswith("gemini-2.5-pro"): thinking = types.ThinkingConfig(thinking_budget=128) @@ -69,7 +110,7 @@ def call_gemini(audio_bytes: bytes, prompt_text: str, model: str, client): elif "pro" in m: thinking = types.ThinkingConfig(thinking_level="low") else: - thinking = types.ThinkingConfig(thinking_level="minimal") + thinking = types.ThinkingConfig(thinking_level=_gemini_floor(model)) config = types.GenerateContentConfig( thinking_config=thinking, temperature=0.0, @@ -87,28 +128,131 @@ def call_gemini(audio_bytes: bytes, prompt_text: str, model: str, client): return content, request_id +def call_fireworks(audio_bytes: bytes, prompt_text: str, model: str, client): + """Fireworks-hosted models (Inkling) take audio as an `audio_url` block with + a data URL — the OpenAI `input_audio` shape is rejected by this endpoint. + Inkling always thinks; "none" is the reasoning floor.""" + b64 = base64.b64encode(audio_bytes).decode("utf-8") + resp = client.chat.completions.create( + model=model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt_text}, + { + "type": "audio_url", + "audio_url": {"url": f"data:audio/wav;base64,{b64}"}, + }, + ], + } + ], + temperature=0.0, + reasoning_effort="none", + ) + content = (resp.choices[0].message.content or "").strip() + return content, getattr(resp, "id", "") or "" + + +def call_openrouter(audio_bytes: bytes, prompt_text: str, model: str, client): + """OpenRouter-hosted audio models. Same audio_url data-URL shape as Fireworks; + reasoning goes through extra_body. See refcoco_multi.call_openrouter for the + host-vs-host caveat on effort="none".""" + b64 = base64.b64encode(audio_bytes).decode("utf-8") + resp = client.chat.completions.create( + model=model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt_text}, + # OpenRouter needs OpenAI's `input_audio` block. The + # `audio_url` data-URI shape that Fireworks requires is + # ACCEPTED here with no error but the audio is silently + # dropped — the model then hallucinates a fluent transcript + # (measured: exact match with input_audio vs unrelated text + # with audio_url, i.e. WER ~1.0 across the whole set). + { + "type": "input_audio", + "input_audio": {"data": b64, "format": "wav"}, + }, + ], + } + ], + temperature=0.0, + extra_body={"reasoning": {"effort": OPENROUTER_EFFORT}}, + ) + return (resp.choices[0].message.content or "").strip(), getattr(resp, "id", "") or "" + + def build_client(provider: str, env: dict): if provider == "gemini": from google import genai - return genai.Client(api_key=env["GEMINI_KEY"]) + + api_key = ( + env.get("GEMINI_KEY") + or os.getenv("GEMINI_API_KEY") + or os.getenv("GEMINI_KEY") + or os.getenv("GOOGLE_API_KEY") + ) + if not api_key: + raise RuntimeError("GEMINI_API_KEY missing from .env") + return genai.Client(api_key=api_key) + if provider == "openrouter": + from openai import OpenAI + + api_key = ( + env.get("OPENROUTER_API_KEY") + or os.getenv("OPENROUTER_API_KEY") + or os.getenv("openrouter_api_key") + ) + if not api_key: + raise RuntimeError("OPENROUTER_API_KEY missing from .env") + return OpenAI(base_url="https://openrouter.ai/api/v1", api_key=api_key) + if provider == "fireworks": + from openai import OpenAI + + # .env in this repo may spell the key lowercase; accept either casing. + api_key = ( + env.get("FIREWORKS_API_KEY") + or os.getenv("FIREWORKS_API_KEY") + or os.getenv("fireworks_api_key") + ) + if not api_key: + raise RuntimeError("FIREWORKS_API_KEY missing from .env") + return OpenAI(base_url="https://api.fireworks.ai/inference/v1", api_key=api_key) raise ValueError(f"Provider not yet supported here: {provider}") def get_call_fn(provider: str): - return {"gemini": call_gemini}[provider] + return { + "gemini": call_gemini, + "fireworks": call_fireworks, + "openrouter": call_openrouter, + }[provider] # -------- pipeline (mirrors voxpopuli_aa.process_sample, but routed via adapter) -------- -async def process_sample(sample: dict, call_fn, model: str, rate_limiter, - writer: JsonlWriter, progress: dict, provider: str, - client) -> dict | None: + +async def process_sample( + sample: dict, + call_fn, + model: str, + rate_limiter, + writer: JsonlWriter, + progress: dict, + provider: str, + client, +) -> dict | None: last_error: str | None = None try: audio_bytes = await asyncio.to_thread(fetch_audio_bytes, sample["file_name"]) except Exception as e: - tqdm.write(f"[{provider} fetch error] id={sample['id']}: {type(e).__name__}: {e}") + tqdm.write( + f"[{provider} fetch error] id={sample['id']}: {type(e).__name__}: {e}" + ) progress["failed"] += 1 return None @@ -169,7 +313,9 @@ async def process_sample(sample: dict, call_fn, model: str, rate_limiter, await asyncio.sleep(2 ** (attempt - 1)) progress["failed"] += 1 - tqdm.write(f"[{provider} FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") + tqdm.write( + f"[{provider} FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}" + ) return None @@ -191,8 +337,10 @@ async def run(provider: str, model: str, pred_path: Path, limit: int | None): if limit is not None: pending = pending[:limit] print(f"--limit applied: will run at most {limit} sample(s)") - print(f"Resume: {len(done_ids)} completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})") + print( + f"Resume: {len(done_ids)} completed, {len(pending)} remaining " + f"(checkpoint: {pred_path})" + ) if not pending: return @@ -200,14 +348,20 @@ async def run(provider: str, model: str, pred_path: Path, limit: int | None): rate_limiter = RateLimiter(RATE_LIMIT) progress = {"total": len(pending), "done": 0, "failed": 0} - tasks = [process_sample(s, call_fn, model, rate_limiter, writer, progress, provider, client) - for s in pending] + tasks = [ + process_sample( + s, call_fn, model, rate_limiter, writer, progress, provider, client + ) + for s in pending + ] try: await tqdm_asyncio.gather(*tasks, desc=f"{provider}/{model}") except Exception: traceback.print_exc() - print(f"\n[{provider}/{model}] Run finished: {progress['done']}/{progress['total']}, " - f"{progress['failed']} failed.") + print( + f"\n[{provider}/{model}] Run finished: {progress['done']}/{progress['total']}, " + f"{progress['failed']} failed." + ) def run_evaluation(pred_path: Path, metrics_path: Path, provider: str, model: str): @@ -227,8 +381,10 @@ def run_evaluation(pred_path: Path, metrics_path: Path, provider: str, model: st print_summary(metrics) output = { **metrics, - "dataset": DATASET_ID, "split": SPLIT, - "provider": provider, "model": model, + "dataset": DATASET_ID, + "split": SPLIT, + "provider": provider, + "model": model, "rate_limit": RATE_LIMIT, } metrics_path.parent.mkdir(parents=True, exist_ok=True) @@ -238,8 +394,10 @@ def run_evaluation(pred_path: Path, metrics_path: Path, provider: str, model: st def main(): - parser = argparse.ArgumentParser(description="Multi-provider VoxPopuli-Cleaned-AA eval") - parser.add_argument("--provider", required=True, choices=["gemini"]) + parser = argparse.ArgumentParser( + description="Multi-provider VoxPopuli-Cleaned-AA eval" + ) + parser.add_argument("--provider", required=True, choices=["gemini", "fireworks", "openrouter"]) parser.add_argument("--model", required=True) parser.add_argument("--predict-only", action="store_true") parser.add_argument("--evaluate-only", action="store_true") diff --git a/benchmarks/gpqa/gpqa_fireworks.py b/benchmarks/gpqa/gpqa_fireworks.py new file mode 100644 index 0000000..fbb4967 --- /dev/null +++ b/benchmarks/gpqa/gpqa_fireworks.py @@ -0,0 +1,324 @@ +""" +GPQA Diamond benchmark for Fireworks-hosted models (default: Inkling). + +Mirrors benchmarks.gpqa.gpqa_openai exactly — same dataset, same prompt, same +deterministic 4-choice shuffle, same parser, same metrics — but routes calls +through Fireworks' OpenAI-compatible endpoint. + +Inkling always thinks. `--thinking off` maps to reasoning_effort="none", which +is its floor rather than a true disable, so even "off" runs emit reasoning +tokens. `--thinking on` maps to reasoning_effort="high" (or --effort). + +Usage: + uv run -m benchmarks.gpqa.gpqa_fireworks + uv run -m benchmarks.gpqa.gpqa_fireworks --thinking on --effort high + uv run -m benchmarks.gpqa.gpqa_fireworks --model accounts/fireworks/models/inkling-small + uv run -m benchmarks.gpqa.gpqa_fireworks --limit 5 + uv run -m benchmarks.gpqa.gpqa_fireworks --evaluate-only + +Env: FIREWORKS_API_KEY. The dataset (Idavidrein/gpqa) is gated — accept its +terms on HuggingFace and log in (`hf auth login`) before running. +""" + +import re +import sys +import json +import time +import asyncio +import argparse +import traceback +from pathlib import Path + +from datasets import load_dataset +from tqdm import tqdm +from tqdm.asyncio import tqdm_asyncio + +PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(PROJECT_ROOT)) + +# Reuse all the dataset/parsing/metrics plumbing from the OpenAI runner. +from benchmarks.gpqa.gpqa_openai import ( # noqa: E402 + DATASET_ID, + CONFIG, + SPLIT, + PROMPT_TEMPLATE, + JsonlWriter, + load_completed_ids, + load_records, + parse_letter, + build_sample, + compute_metrics, +) +from src.commons_fireworks import INKLING, fireworks_client # noqa: E402 + +RESULTS_DIR = PROJECT_ROOT / "results" + +DEFAULT_MODEL = INKLING +DEFAULT_THINKING = "off" # 'on' or 'off' ('off' = reasoning_effort="none" floor) +DEFAULT_EFFORT: str | None = None # None | low | medium | high | xhigh | max +TEMPERATURE = 0.0 +CONCURRENCY = 10 +MAX_RETRIES = 5 +RETRY_BACKOFF_CAP_S = 30.0 + +MODEL = DEFAULT_MODEL +THINKING = DEFAULT_THINKING +EFFORT: str | None = DEFAULT_EFFORT + + +def model_slug(model: str) -> str: + """accounts/fireworks/models/inkling-small -> inklingsmall""" + return re.sub(r"[^a-z0-9]", "", model.rsplit("/", 1)[-1].lower()) + + +def invoke_fireworks(messages: list[dict]): + effort = "none" if THINKING == "off" else (EFFORT or "high") + return fireworks_client.chat.completions.create( + model=MODEL, + messages=messages, + temperature=TEMPERATURE, + reasoning_effort=effort, + ) + + +def _reasoning_tokens(response) -> int | None: + u = getattr(response, "usage", None) + if u is None: + return None + details = getattr(u, "completion_tokens_details", None) + val = getattr(details, "reasoning_tokens", None) if details else None + if val is None: + # Fireworks reports it at the top level of `usage` for Inkling. + val = getattr(u, "reasoning_tokens", None) + try: + return int(val) if val is not None else None + except (TypeError, ValueError): + return None + + +async def process_sample( + sample: dict, semaphore: asyncio.Semaphore, writer: JsonlWriter, progress: dict +) -> dict | None: + prompt = PROMPT_TEMPLATE.format( + question=sample["question"], + a=sample["a"], + b=sample["b"], + c=sample["c"], + d=sample["d"], + ) + messages = [{"role": "user", "content": prompt}] + last_error: str | None = None + # Recorded per sample: stdout can be truncated by whatever captures it, and + # "why did half the run retry" is only answerable from the checkpoint. + retry_errors: list[str] = [] + + for attempt in range(1, MAX_RETRIES + 1): + start = time.perf_counter() + try: + async with semaphore: + start = time.perf_counter() + response = await asyncio.to_thread(invoke_fireworks, messages) + latency_ms = int((time.perf_counter() - start) * 1000) + content = (response.choices[0].message.content or "").strip() + request_id = getattr(response, "id", None) + if not content: + last_error = "empty response content" + raise RuntimeError(last_error) + + predicted = parse_letter(content) + correct = predicted == sample["correct_letter"] + reasoning_tokens = _reasoning_tokens(response) + + record = { + "id": sample["id"], + "domain": sample["domain"], + "subdomain": sample["subdomain"], + "correct_letter": sample["correct_letter"], + "prediction": predicted, + "correct": correct, + "response": content, + "request_id": request_id, + "latency_ms": latency_ms, + "attempts": attempt, + "reasoning_tokens": reasoning_tokens, + "retry_errors": retry_errors, + } + await writer.append(record) + + progress["done"] += 1 + if correct: + progress["correct"] += 1 + mark = "OK" if correct else "X " + tqdm.write( + f"[{progress['done']}/{progress['total']}] {mark} " + f"id={sample['id']} domain={sample['domain']:10} " + f"gold={sample['correct_letter']} pred={predicted or '?'} " + f"latency={latency_ms}ms reasoning_tok={reasoning_tokens if reasoning_tokens is not None else '?'} " + f"attempt={attempt}" + ) + return record + + except Exception as e: + latency_ms = int((time.perf_counter() - start) * 1000) + last_error = f"{type(e).__name__}: {e}" + retry_errors.append(last_error[:200]) + tqdm.write( + f"[error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " + f"latency={latency_ms}ms error={last_error}" + ) + if attempt < MAX_RETRIES: + await asyncio.sleep(min(2 ** (attempt - 1), RETRY_BACKOFF_CAP_S)) + + progress["failed"] += 1 + tqdm.write(f"[FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") + return None + + +def print_summary(metrics: dict): + print(f"\n{'=' * 60}") + print(f"GPQA Diamond — {DATASET_ID}/{CONFIG} ({MODEL}, thinking={THINKING})") + print(f"{'=' * 60}") + print( + f"Accuracy : {metrics['accuracy']:.4f} ({metrics['correct']}/{metrics['total']})" + ) + print(f"Unparseable: {metrics['unparseable']}") + print("\nPer high-level domain:") + for d in sorted(metrics["per_domain"]): + v = metrics["per_domain"][d] + print(f" {d:12} n={v['n']:>3} acc={v['accuracy']:.4f}") + if metrics.get("latency"): + lat = metrics["latency"] + print( + f"\nLatency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " + f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms max={lat['max_ms']}ms" + ) + + +async def run_predictions(pred_path: Path, limit: int | None): + print(f"Loading {DATASET_ID}/{CONFIG} (split={SPLIT})...") + ds = load_dataset(DATASET_ID, CONFIG, split=SPLIT) + print(f"Loaded {len(ds)} rows") + samples = [build_sample(dict(row)) for row in ds] + + done_ids = load_completed_ids(pred_path) + pending = [s for s in samples if s["id"] not in done_ids] + if limit is not None: + pending = pending[:limit] + print(f"--limit applied: will run at most {limit} sample(s)") + print( + f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " + f"(checkpoint: {pred_path})" + ) + if not pending: + return + + writer = JsonlWriter(pred_path) + semaphore = asyncio.Semaphore(CONCURRENCY) + progress = {"total": len(pending), "done": 0, "correct": 0, "failed": 0} + + tasks = [process_sample(s, semaphore, writer, progress) for s in pending] + try: + await tqdm_asyncio.gather(*tasks, desc=f"GPQA Diamond / {MODEL}") + except Exception: + traceback.print_exc() + acc = progress["correct"] / progress["done"] if progress["done"] else 0.0 + print( + f"\nRun finished: {progress['done']}/{progress['total']} answered, " + f"{progress['correct']} correct (acc={acc:.4f}), {progress['failed']} failed." + ) + + +def run_evaluation(pred_path: Path, metrics_path: Path): + if not pred_path.exists(): + print(f"No predictions found at {pred_path}") + sys.exit(1) + results = load_records(pred_path) + if not results: + print(f"No records in {pred_path}") + sys.exit(1) + for r in results: + if r.get("prediction") is None and r.get("response"): + r["prediction"] = parse_letter(r["response"]) + if r.get("correct") is None and r.get("prediction") is not None: + r["correct"] = r["prediction"] == r.get("correct_letter") + + metrics = compute_metrics(results) + print_summary(metrics) + reasoning_toks = [ + r["reasoning_tokens"] for r in results if r.get("reasoning_tokens") is not None + ] + output = { + **metrics, + "dataset": DATASET_ID, + "config": CONFIG, + "split": SPLIT, + "model": MODEL, + "thinking": THINKING, + "effort": "none" if THINKING == "off" else (EFFORT or "high"), + "temperature": TEMPERATURE, + "concurrency": CONCURRENCY, + "provider": "fireworks", + "mean_reasoning_tokens": ( + sum(reasoning_toks) / len(reasoning_toks) if reasoning_toks else None + ), + } + metrics_path.parent.mkdir(parents=True, exist_ok=True) + with open(metrics_path, "w") as f: + json.dump(output, f, indent=2, ensure_ascii=False) + print(f"\nMetrics saved to {metrics_path}") + + +def main(): + global MODEL, THINKING, EFFORT + parser = argparse.ArgumentParser( + description="GPQA Diamond benchmark for Fireworks-hosted models" + ) + parser.add_argument( + "--model", + default=DEFAULT_MODEL, + help="Fireworks model id (accounts/fireworks/...)", + ) + parser.add_argument( + "--thinking", + default=DEFAULT_THINKING, + choices=["on", "off"], + help="off = reasoning_effort 'none' (Inkling's floor); on = 'high' or --effort", + ) + parser.add_argument( + "--effort", + default=DEFAULT_EFFORT, + choices=["low", "medium", "high", "xhigh", "max"], + help="Explicit reasoning_effort. Only applied when thinking=on.", + ) + parser.add_argument("--predict-only", action="store_true") + parser.add_argument("--evaluate-only", action="store_true") + parser.add_argument( + "--limit", + type=int, + default=None, + help="Only run the first N unanswered samples", + ) + args = parser.parse_args() + + MODEL = args.model + THINKING = args.thinking + EFFORT = args.effort + + effort_suffix = f"_effort{EFFORT}" if (THINKING == "on" and EFFORT) else "" + tag = ( + f"fireworks_{model_slug(MODEL)}_thinking{THINKING}{effort_suffix}_gpqa_diamond" + ) + pred_path = RESULTS_DIR / f"{tag}_responses.jsonl" + metrics_path = RESULTS_DIR / f"{tag}_metrics.json" + + if args.evaluate_only: + run_evaluation(pred_path, metrics_path) + elif args.predict_only: + asyncio.run(run_predictions(pred_path, limit=args.limit)) + else: + asyncio.run(run_predictions(pred_path, limit=args.limit)) + run_evaluation(pred_path, metrics_path) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/gpqa/gpqa_openai.py b/benchmarks/gpqa/gpqa_openai.py index 199dc04..fe62940 100644 --- a/benchmarks/gpqa/gpqa_openai.py +++ b/benchmarks/gpqa/gpqa_openai.py @@ -81,16 +81,15 @@ ) OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") -if not OPENAI_API_KEY: - raise RuntimeError( - "OPENAI_API_KEY is not set. Add it to .env " - "(get one from https://platform.openai.com/api-keys)." - ) -# Bypass the .env's interfaze base_url override. -openai_client = OpenAI( - base_url="https://api.openai.com/v1", - api_key=OPENAI_API_KEY, +# Bypass the .env's interfaze base_url override. Built only when the key is +# present — the openrouter/fireworks runners import this module's dataset and +# scoring helpers, so a hard raise here would block them; invoke_openai raises +# instead. +openai_client = ( + OpenAI(base_url="https://api.openai.com/v1", api_key=OPENAI_API_KEY) + if OPENAI_API_KEY + else None ) @@ -99,6 +98,11 @@ def model_slug(model: str) -> str: def invoke_openai(messages: list[dict]): + if openai_client is None: + raise RuntimeError( + "OPENAI_API_KEY is not set. Add it to .env " + "(get one from https://platform.openai.com/api-keys)." + ) # GPT-5.x rejects temperature!=default when reasoning is engaged # ("Unsupported value: 'temperature' does not support 0.0 with this model. # Only the default (1) value is supported."). With reasoning off ('none'), @@ -216,11 +220,15 @@ def build_sample(row: dict) -> dict: } -async def process_sample(sample: dict, semaphore: asyncio.Semaphore, - writer: JsonlWriter, progress: dict) -> dict | None: +async def process_sample( + sample: dict, semaphore: asyncio.Semaphore, writer: JsonlWriter, progress: dict +) -> dict | None: prompt = PROMPT_TEMPLATE.format( question=sample["question"], - a=sample["a"], b=sample["b"], c=sample["c"], d=sample["d"], + a=sample["a"], + b=sample["b"], + c=sample["c"], + d=sample["d"], ) messages = [{"role": "user", "content": prompt}] last_error: str | None = None @@ -300,13 +308,16 @@ def compute_metrics(results: list[dict]) -> dict: for d, rows in by_domain.items() } - latencies = [r["latency_ms"] for r in results if isinstance(r.get("latency_ms"), int)] + latencies = [ + r["latency_ms"] for r in results if isinstance(r.get("latency_ms"), int) + ] latency_stats = {} if latencies: lats = sorted(latencies) n = len(lats) latency_stats = { - "count": n, "mean_ms": sum(lats) / n, + "count": n, + "mean_ms": sum(lats) / n, "p50_ms": lats[n // 2], "p90_ms": lats[min(n - 1, int(n * 0.9))], "p99_ms": lats[min(n - 1, int(n * 0.99))], @@ -325,9 +336,13 @@ def compute_metrics(results: list[dict]) -> dict: def print_summary(metrics: dict): print(f"\n{'=' * 60}") - print(f"GPQA Diamond — {DATASET_ID}/{CONFIG} ({MODEL}, reasoning={REASONING_EFFORT})") + print( + f"GPQA Diamond — {DATASET_ID}/{CONFIG} ({MODEL}, reasoning={REASONING_EFFORT})" + ) print(f"{'=' * 60}") - print(f"Accuracy : {metrics['accuracy']:.4f} ({metrics['correct']}/{metrics['total']})") + print( + f"Accuracy : {metrics['accuracy']:.4f} ({metrics['correct']}/{metrics['total']})" + ) print(f"Unparseable: {metrics['unparseable']}") print("\nPer high-level domain:") for d in sorted(metrics["per_domain"]): @@ -335,8 +350,10 @@ def print_summary(metrics: dict): print(f" {d:12} n={v['n']:>3} acc={v['accuracy']:.4f}") if metrics.get("latency"): lat = metrics["latency"] - print(f"\nLatency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " - f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms max={lat['max_ms']}ms") + print( + f"\nLatency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " + f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms max={lat['max_ms']}ms" + ) async def run_predictions(pred_path: Path, limit: int | None): @@ -350,8 +367,10 @@ async def run_predictions(pred_path: Path, limit: int | None): if limit is not None: pending = pending[:limit] print(f"--limit applied: will run at most {limit} sample(s)") - print(f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})") + print( + f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " + f"(checkpoint: {pred_path})" + ) if not pending: return @@ -365,8 +384,10 @@ async def run_predictions(pred_path: Path, limit: int | None): except Exception: traceback.print_exc() acc = progress["correct"] / progress["done"] if progress["done"] else 0.0 - print(f"\nRun finished: {progress['done']}/{progress['total']} answered, " - f"{progress['correct']} correct (acc={acc:.4f}), {progress['failed']} failed.") + print( + f"\nRun finished: {progress['done']}/{progress['total']} answered, " + f"{progress['correct']} correct (acc={acc:.4f}), {progress['failed']} failed." + ) def run_evaluation(pred_path: Path, metrics_path: Path): @@ -403,15 +424,27 @@ def run_evaluation(pred_path: Path, metrics_path: Path): def main(): global MODEL, REASONING_EFFORT - parser = argparse.ArgumentParser(description="GPQA Diamond benchmark for OpenAI GPT-5.x") - parser.add_argument("--model", default=DEFAULT_MODEL, - help="OpenAI model id (e.g. gpt-5.5, gpt-5.4-mini)") - parser.add_argument("--reasoning-effort", default=DEFAULT_REASONING_EFFORT, - help="reasoning_effort param ('none', 'minimal', 'low', 'medium', 'high', etc.)") + parser = argparse.ArgumentParser( + description="GPQA Diamond benchmark for OpenAI GPT-5.x" + ) + parser.add_argument( + "--model", + default=DEFAULT_MODEL, + help="OpenAI model id (e.g. gpt-5.5, gpt-5.4-mini)", + ) + parser.add_argument( + "--reasoning-effort", + default=DEFAULT_REASONING_EFFORT, + help="reasoning_effort param ('none', 'minimal', 'low', 'medium', 'high', etc.)", + ) parser.add_argument("--predict-only", action="store_true") parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument("--limit", type=int, default=None, - help="Only run the first N unanswered samples") + parser.add_argument( + "--limit", + type=int, + default=None, + help="Only run the first N unanswered samples", + ) args = parser.parse_args() MODEL = args.model diff --git a/benchmarks/gpqa/gpqa_openrouter.py b/benchmarks/gpqa/gpqa_openrouter.py index 3603c74..31ed0ed 100644 --- a/benchmarks/gpqa/gpqa_openrouter.py +++ b/benchmarks/gpqa/gpqa_openrouter.py @@ -61,7 +61,9 @@ THINKING = DEFAULT_THINKING EFFORT: str | None = DEFAULT_EFFORT -OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY") +OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY") or os.getenv( + "openrouter_api_key" +) if not OPENROUTER_API_KEY: raise RuntimeError( "OPENROUTER_API_KEY is not set. Add it to .env " diff --git a/benchmarks/mmmlu/mmmlu_multi.py b/benchmarks/mmmlu/mmmlu_multi.py index bc11c9e..71877ca 100644 --- a/benchmarks/mmmlu/mmmlu_multi.py +++ b/benchmarks/mmmlu/mmmlu_multi.py @@ -20,6 +20,7 @@ uv run -m benchmarks.mmmlu.mmmlu_multi --provider openai --model gpt-5.4-mini uv run -m benchmarks.mmmlu.mmmlu_multi --provider gemini --model gemini-3.1-pro-preview uv run -m benchmarks.mmmlu.mmmlu_multi --provider anthropic --model claude-sonnet-4-6 + uv run -m benchmarks.mmmlu.mmmlu_multi --provider fireworks --model accounts/fireworks/models/inkling # Smoke (1 sample per language = 14 total): uv run -m benchmarks.mmmlu.mmmlu_multi --provider gemini --model gemini-3-flash-preview --limit 1 @@ -86,6 +87,28 @@ def _load_interfaze_env_fallback() -> None: RESULTS_DIR = PROJECT_ROOT / "results" RATE_LIMIT = 50 MAX_RETRIES = 3 + +# Cap on requests in flight. RATE_LIMIT bounds how many requests *start* per +# second but not how many are outstanding: gather() launches every pending +# sample at once, so hundreds pile up, queue server-side, and eventually 429 +# after tens of minutes of latency. Override with BENCH_MAX_IN_FLIGHT. +MAX_IN_FLIGHT = int(os.getenv("BENCH_MAX_IN_FLIGHT", "8")) + +def _gemini_floor(model: str) -> str: + """Lowest thinking_level the given Gemini model actually accepts. + + gemini-3.7+ flash rejects MINIMAL server-side ("Thinking level MINIMAL is + not supported for this model"), and the SDK enum has no "off"/"none", so + "low" is its true floor. Older 3.x flash models still take "minimal". + BENCH_GEMINI_FLOOR overrides for anything not covered here. + """ + override = os.getenv("BENCH_GEMINI_FLOOR") + if override: + return override + m = model.lower() + if any(v in m for v in ("3.7", "3.8", "3.9", "4.")): + return "low" + return "minimal" DEFAULT_TEMPERATURE = 0.0 # Dataset variant — "full" (openai/MMMLU, ~196k) or "lite" (opencompass/mmmlu_lite, @@ -110,6 +133,7 @@ def _load_interfaze_env_fallback() -> None: # Dataset loading + sample building # --------------------------------------------------------------------------- + def build_sample_lite(row: dict, language: str, row_index: int) -> dict: """opencompass/mmmlu_lite uses input/target/A-D/subject (no Unnamed: 0).""" return { @@ -132,7 +156,9 @@ def load_dataset_for_variant(variant: str, lang: str): return load_dataset(DATASET_FULL_ID, lang, split="test") -def build_sample_for_variant(variant: str, row: dict, language: str, row_index: int) -> dict: +def build_sample_for_variant( + variant: str, row: dict, language: str, row_index: int +) -> dict: if variant == "lite": return build_sample_lite(row, language, row_index) return build_sample(row, language) @@ -147,12 +173,14 @@ def build_sample_for_variant(variant: str, row: dict, language: str, row_index: # Run synchronously inside asyncio.to_thread. # --------------------------------------------------------------------------- + def _safe_int(x): try: return int(x) if x is not None else None except (TypeError, ValueError): return None + def _openai_usage(resp) -> dict: """Extract input/output/reasoning tokens from an OpenAI chat.completions response.""" u = getattr(resp, "usage", None) @@ -160,6 +188,9 @@ def _openai_usage(resp) -> dict: return {"input_tokens": None, "output_tokens": None, "reasoning_tokens": None} details = getattr(u, "completion_tokens_details", None) reasoning = getattr(details, "reasoning_tokens", None) if details else None + if reasoning is None: + # Fireworks reports it at the top level of `usage` for Inkling. + reasoning = getattr(u, "reasoning_tokens", None) return { "input_tokens": _safe_int(getattr(u, "prompt_tokens", None)), "output_tokens": _safe_int(getattr(u, "completion_tokens", None)), @@ -243,7 +274,10 @@ def call_anthropic(prompt: str, model: str, client) -> tuple[str, str | None, di kwargs["temperature"] = DEFAULT_TEMPERATURE kwargs["max_tokens"] = ANTHROPIC_OFF_MAX_TOKENS else: - kwargs["thinking"] = {"type": "enabled", "budget_tokens": ANTHROPIC_HIGH_BUDGET_TOKENS} + kwargs["thinking"] = { + "type": "enabled", + "budget_tokens": ANTHROPIC_HIGH_BUDGET_TOKENS, + } kwargs["max_tokens"] = ANTHROPIC_HIGH_MAX_TOKENS resp = client.messages.create(**kwargs) parts = [b.text for b in resp.content if getattr(b, "type", None) == "text"] @@ -265,6 +299,12 @@ def _openrouter_extra_body(model: str) -> dict: if REASONING_MODE == "off": return {"reasoning": {"effort": "minimal"}} return {"reasoning": {"enabled": True}} + if m.startswith("google/"): + # Gemini rejects reasoning.enabled=false outright ("Reasoning is + # mandatory for this endpoint and cannot be disabled"), so "off" maps to + # its real floor: effort=minimal, measured at 0 reasoning tokens. This + # matches what call_gemini does natively (thinking_level="minimal"). + return {"reasoning": {"effort": "minimal" if REASONING_MODE == "off" else "high"}} if m.startswith("moonshotai/"): body = {"reasoning": {"enabled": REASONING_MODE != "off"}} body["provider"] = {"only": ["moonshotai"]} @@ -288,6 +328,24 @@ def call_openrouter(prompt: str, model: str, client) -> tuple[str, str | None, d ) +def call_fireworks(prompt: str, model: str, client) -> tuple[str, str | None, dict]: + """Fireworks-hosted models (Inkling). Inkling always thinks — reasoning_effort + accepts none|low|medium|high|xhigh|max and "none" is the floor, not a true + off, so `reasoningoff` runs here still emit reasoning tokens (recorded per + sample in the output JSONL).""" + resp = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}], + temperature=DEFAULT_TEMPERATURE, + reasoning_effort="none" if REASONING_MODE == "off" else "high", + ) + return ( + (resp.choices[0].message.content or "").strip(), + getattr(resp, "id", None), + _openai_usage(resp), + ) + + def call_gemini(prompt: str, model: str, client) -> tuple[str, str | None, dict]: """Gemini — off goes to each model's floor; high goes to max thinking. 3.x Pro: 'low' floor / 'high' max. @@ -296,6 +354,7 @@ def call_gemini(prompt: str, model: str, client) -> tuple[str, str | None, dict] 2.5 Flash: budget=0 floor / budget=-1 for high. Temperature 0 is fine with thinking on for Gemini.""" from google.genai import types + m = model.lower() if REASONING_MODE == "off": if m.startswith("gemini-2.5-pro"): @@ -305,7 +364,7 @@ def call_gemini(prompt: str, model: str, client) -> tuple[str, str | None, dict] elif "pro" in m: thinking = types.ThinkingConfig(thinking_level="low") else: - thinking = types.ThinkingConfig(thinking_level="minimal") + thinking = types.ThinkingConfig(thinking_level=_gemini_floor(model)) else: if m.startswith("gemini-2.5"): thinking = types.ThinkingConfig(thinking_budget=-1) @@ -330,6 +389,7 @@ def call_gemini(prompt: str, model: str, client) -> tuple[str, str | None, dict] def build_client(provider: str): if provider == "interfaze": from openai import OpenAI + api_key = os.getenv("INTERFAZE_API_KEY") if not api_key: raise RuntimeError("INTERFAZE_API_KEY missing from .env") @@ -339,6 +399,7 @@ def build_client(provider: str): return OpenAI(base_url="https://api.interfaze.ai/v1", api_key=api_key) if provider == "openai": from openai import OpenAI + api_key = os.getenv("OPENAI_API_KEY") if not api_key: raise RuntimeError("OPENAI_API_KEY missing from .env") @@ -346,24 +407,41 @@ def build_client(provider: str): return OpenAI(base_url="https://api.openai.com/v1", api_key=api_key) if provider == "anthropic": from anthropic import Anthropic + api_key = os.getenv("ANTHROPIC_API_KEY") if not api_key: raise RuntimeError("ANTHROPIC_API_KEY missing from .env") return Anthropic(api_key=api_key) if provider == "gemini": from google import genai + api_key = ( - os.getenv("GEMINI_KEY") or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY") + os.getenv("GEMINI_KEY") + or os.getenv("GEMINI_API_KEY") + or os.getenv("GOOGLE_API_KEY") ) if not api_key: raise RuntimeError("GEMINI_KEY missing from .env") return genai.Client(api_key=api_key) if provider == "openrouter": from openai import OpenAI - api_key = os.getenv("OPENROUTER_API_KEY") or os.getenv("OPENROUTER_KEY") + + api_key = ( + os.getenv("OPENROUTER_API_KEY") + or os.getenv("openrouter_api_key") + or os.getenv("OPENROUTER_KEY") + ) if not api_key: raise RuntimeError("OPENROUTER_API_KEY missing from .env") return OpenAI(base_url="https://openrouter.ai/api/v1", api_key=api_key) + if provider == "fireworks": + from openai import OpenAI + + # .env in this repo may spell the key lowercase; accept either casing. + api_key = os.getenv("FIREWORKS_API_KEY") or os.getenv("fireworks_api_key") + if not api_key: + raise RuntimeError("FIREWORKS_API_KEY missing from .env") + return OpenAI(base_url="https://api.fireworks.ai/inference/v1", api_key=api_key) raise ValueError(f"unknown provider: {provider}") @@ -374,6 +452,7 @@ def get_call_fn(provider: str): "anthropic": call_anthropic, "gemini": call_gemini, "openrouter": call_openrouter, + "fireworks": call_fireworks, }[provider] @@ -385,12 +464,23 @@ def model_slug(model: str) -> str: # Pipeline # --------------------------------------------------------------------------- -async def process_sample(sample: dict, call_fn, model: str, rate_limiter: RateLimiter, - writer: JsonlWriter, progress: dict, provider: str, - client) -> dict | None: + +async def process_sample( + sample: dict, + call_fn, + model: str, + rate_limiter: RateLimiter, + writer: JsonlWriter, + progress: dict, + provider: str, + client, +) -> dict | None: prompt = PROMPT_TEMPLATE.format( question=sample["question"], - a=sample["a"], b=sample["b"], c=sample["c"], d=sample["d"], + a=sample["a"], + b=sample["b"], + c=sample["c"], + d=sample["d"], ) last_error: str | None = None @@ -398,7 +488,9 @@ async def process_sample(sample: dict, call_fn, model: str, rate_limiter: RateLi await rate_limiter.acquire() start = time.perf_counter() try: - content, request_id, usage = await asyncio.to_thread(call_fn, prompt, model, client) + content, request_id, usage = await asyncio.to_thread( + call_fn, prompt, model, client + ) latency_ms = int((time.perf_counter() - start) * 1000) if not content: last_error = "empty response content" @@ -454,7 +546,9 @@ async def process_sample(sample: dict, call_fn, model: str, rate_limiter: RateLi await asyncio.sleep(2 ** (attempt - 1)) progress["failed"] += 1 - tqdm.write(f"[{provider}/{model} FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") + tqdm.write( + f"[{provider}/{model} FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}" + ) return None @@ -476,8 +570,9 @@ def load_all_samples(languages: list[str], limit: int | None) -> list[dict]: return all_samples -async def run(provider: str, model: str, languages: list[str], pred_path: Path, - limit: int | None): +async def run( + provider: str, model: str, languages: list[str], pred_path: Path, limit: int | None +): client = build_client(provider) call_fn = get_call_fn(provider) @@ -485,8 +580,10 @@ async def run(provider: str, model: str, languages: list[str], pred_path: Path, done_ids = load_completed_ids(pred_path) pending = [s for s in samples if s["id"] not in done_ids] print(f"[{provider}/{model}] Total samples: {len(samples)}") - print(f"[{provider}/{model}] Resume: {len(done_ids)} done, {len(pending)} pending " - f"(checkpoint: {pred_path})") + print( + f"[{provider}/{model}] Resume: {len(done_ids)} done, {len(pending)} pending " + f"(checkpoint: {pred_path})" + ) if not pending: return @@ -500,8 +597,15 @@ async def run(provider: str, model: str, languages: list[str], pred_path: Path, "failed": 0, } - tasks = [process_sample(s, call_fn, model, rate_limiter, writer, progress, provider, client) - for s in pending] + sem = asyncio.Semaphore(MAX_IN_FLIGHT) + + async def bounded(sample): + async with sem: + return await process_sample( + sample, call_fn, model, rate_limiter, writer, progress, provider, client + ) + + tasks = [bounded(s) for s in pending] try: await tqdm_asyncio.gather(*tasks, desc=f"{provider}/{model}") except Exception: @@ -551,17 +655,41 @@ def run_evaluation(pred_path: Path, metrics_path: Path, provider: str, model: st def main(): global REASONING_MODE, DATASET_VARIANT parser = argparse.ArgumentParser(description="Multi-provider MMMLU runner") - parser.add_argument("--provider", required=True, - choices=["interfaze", "openai", "anthropic", "gemini", "openrouter"]) + parser.add_argument( + "--provider", + required=True, + choices=[ + "interfaze", + "openai", + "anthropic", + "gemini", + "openrouter", + "fireworks", + ], + ) parser.add_argument("--model", required=True, help="Provider-specific model id") - parser.add_argument("--reasoning", default="off", choices=["off", "high"], - help="off = each model at its floor; high = each at max thinking") - parser.add_argument("--dataset-variant", default="lite", choices=["lite", "full"], - help="lite = opencompass/mmmlu_lite (~20k); full = openai/MMMLU (~196k)") - parser.add_argument("--languages", nargs="+", default=LANGUAGES, choices=LANGUAGES, - help="Subset of languages (default: all 14)") - parser.add_argument("--limit", type=int, default=None, - help="Cap samples per language (smoke test)") + parser.add_argument( + "--reasoning", + default="off", + choices=["off", "high"], + help="off = each model at its floor; high = each at max thinking", + ) + parser.add_argument( + "--dataset-variant", + default="lite", + choices=["lite", "full"], + help="lite = opencompass/mmmlu_lite (~20k); full = openai/MMMLU (~196k)", + ) + parser.add_argument( + "--languages", + nargs="+", + default=LANGUAGES, + choices=LANGUAGES, + help="Subset of languages (default: all 14)", + ) + parser.add_argument( + "--limit", type=int, default=None, help="Cap samples per language (smoke test)" + ) parser.add_argument("--predict-only", action="store_true") parser.add_argument("--evaluate-only", action="store_true") args = parser.parse_args() @@ -576,9 +704,13 @@ def main(): if args.evaluate_only: run_evaluation(pred_path, metrics_path, args.provider, args.model) elif args.predict_only: - asyncio.run(run(args.provider, args.model, args.languages, pred_path, limit=args.limit)) + asyncio.run( + run(args.provider, args.model, args.languages, pred_path, limit=args.limit) + ) else: - asyncio.run(run(args.provider, args.model, args.languages, pred_path, limit=args.limit)) + asyncio.run( + run(args.provider, args.model, args.languages, pred_path, limit=args.limit) + ) run_evaluation(pred_path, metrics_path, args.provider, args.model) diff --git a/benchmarks/mmmu_pro/mmmu_pro_multi.py b/benchmarks/mmmu_pro/mmmu_pro_multi.py index 907e152..293f718 100644 --- a/benchmarks/mmmu_pro/mmmu_pro_multi.py +++ b/benchmarks/mmmu_pro/mmmu_pro_multi.py @@ -79,6 +79,28 @@ def _load_interfaze_env_fallback() -> None: RATE_LIMIT = 25 MAX_RETRIES = 3 +# Cap on requests in flight. RATE_LIMIT bounds how many requests *start* per +# second but not how many are outstanding: gather() launches every pending +# sample at once, so hundreds pile up, queue server-side, and eventually 429 +# after tens of minutes of latency. Override with BENCH_MAX_IN_FLIGHT. +MAX_IN_FLIGHT = int(os.getenv("BENCH_MAX_IN_FLIGHT", "8")) + +def _gemini_floor(model: str) -> str: + """Lowest thinking_level the given Gemini model actually accepts. + + gemini-3.7+ flash rejects MINIMAL server-side ("Thinking level MINIMAL is + not supported for this model"), and the SDK enum has no "off"/"none", so + "low" is its true floor. Older 3.x flash models still take "minimal". + BENCH_GEMINI_FLOOR overrides for anything not covered here. + """ + override = os.getenv("BENCH_GEMINI_FLOOR") + if override: + return override + m = model.lower() + if any(v in m for v in ("3.7", "3.8", "3.9", "4.")): + return "low" + return "minimal" + ANTHROPIC_OFF_MAX_TOKENS = 1024 ANTHROPIC_HIGH_BUDGET_TOKENS = 10_000 ANTHROPIC_HIGH_MAX_TOKENS = 16_000 @@ -94,6 +116,7 @@ def _load_interfaze_env_fallback() -> None: # IO helpers # --------------------------------------------------------------------------- + class JsonlWriter: def __init__(self, path: Path): self.path = path @@ -168,6 +191,7 @@ def load_records(path: Path) -> list[dict]: # Dataset / sample building # --------------------------------------------------------------------------- + def parse_options(raw: str) -> list[str]: """`options` is a stringified Python list. Use literal_eval (safe).""" if isinstance(raw, list): @@ -220,6 +244,7 @@ def load_samples(setting: str, limit: int | None) -> list[dict]: # Image preprocessing # --------------------------------------------------------------------------- + def image_to_jpeg_bytes(image: Image.Image) -> bytes: if image.mode != "RGB": image = image.convert("RGB") @@ -236,6 +261,7 @@ def image_to_jpeg_bytes(image: Image.Image) -> bytes: # Prompt construction # --------------------------------------------------------------------------- + def build_options_block(options: list[str]) -> str: lines = [] for letter, opt in zip(LETTERS_10, options): @@ -245,7 +271,7 @@ def build_options_block(options: list[str]) -> str: PROMPT_STANDARD = ( "Answer the following multiple-choice question. The question may reference " - "images via tags like \"\", \"\". The corresponding images " + 'images via tags like "", "". The corresponding images ' "are attached in order.\n\n" "Respond with ONLY a single letter (A through J) corresponding to the correct " "option. Do not explain.\n\n" @@ -289,6 +315,7 @@ def parse_answer(text: str) -> str | None: # Provider adapters # --------------------------------------------------------------------------- + def _safe_int(x): try: return int(x) if x is not None else None @@ -302,6 +329,9 @@ def _openai_usage(resp) -> dict: return {"input_tokens": None, "output_tokens": None, "reasoning_tokens": None} details = getattr(u, "completion_tokens_details", None) rt = getattr(details, "reasoning_tokens", None) if details else None + if rt is None: + # Fireworks reports it at the top level of `usage` for Inkling. + rt = getattr(u, "reasoning_tokens", None) return { "input_tokens": _safe_int(getattr(u, "prompt_tokens", None)), "output_tokens": _safe_int(getattr(u, "completion_tokens", None)), @@ -351,10 +381,12 @@ def _openai_style_content(prompt: str, images: list[Image.Image]) -> list[dict]: content: list[dict] = [{"type": "text", "text": prompt}] for img in images: b64 = base64.b64encode(image_to_jpeg_bytes(img)).decode("utf-8") - content.append({ - "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64}"}, - }) + content.append( + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{b64}"}, + } + ) return content @@ -398,10 +430,12 @@ def call_anthropic(sample: dict, model: str, client) -> tuple[str, str | None, d parts: list[dict] = [] for img in _images_for_sample(sample): b64 = base64.b64encode(image_to_jpeg_bytes(img)).decode("utf-8") - parts.append({ - "type": "image", - "source": {"type": "base64", "media_type": "image/jpeg", "data": b64}, - }) + parts.append( + { + "type": "image", + "source": {"type": "base64", "media_type": "image/jpeg", "data": b64}, + } + ) parts.append({"type": "text", "text": prompt}) kwargs = { "model": model, @@ -412,7 +446,10 @@ def call_anthropic(sample: dict, model: str, client) -> tuple[str, str | None, d kwargs["temperature"] = DEFAULT_TEMPERATURE kwargs["max_tokens"] = ANTHROPIC_OFF_MAX_TOKENS else: - kwargs["thinking"] = {"type": "enabled", "budget_tokens": ANTHROPIC_HIGH_BUDGET_TOKENS} + kwargs["thinking"] = { + "type": "enabled", + "budget_tokens": ANTHROPIC_HIGH_BUDGET_TOKENS, + } kwargs["max_tokens"] = ANTHROPIC_HIGH_MAX_TOKENS resp = client.messages.create(**kwargs) text_parts = [b.text for b in resp.content if getattr(b, "type", None) == "text"] @@ -427,6 +464,12 @@ def _openrouter_extra_body(model: str) -> dict: if REASONING_MODE == "off": return {"reasoning": {"effort": "minimal"}} return {"reasoning": {"enabled": True}} + if m.startswith("google/"): + # Gemini rejects reasoning.enabled=false outright ("Reasoning is + # mandatory for this endpoint and cannot be disabled"), so "off" maps to + # its real floor: effort=minimal, measured at 0 reasoning tokens. This + # matches what call_gemini does natively (thinking_level="minimal"). + return {"reasoning": {"effort": "minimal" if REASONING_MODE == "off" else "high"}} if m.startswith("moonshotai/"): body = {"reasoning": {"enabled": REASONING_MODE != "off"}} body["provider"] = {"only": ["moonshotai"]} @@ -451,8 +494,28 @@ def call_openrouter(sample: dict, model: str, client) -> tuple[str, str | None, ) +def call_fireworks(sample: dict, model: str, client) -> tuple[str, str | None, dict]: + """Fireworks-hosted models (Inkling). Images go in as OpenAI-shaped + image_url data URLs. Inkling always thinks — "none" is the reasoning floor, + not a true off; per-sample reasoning_tokens are recorded so the actual + thinking spend is auditable.""" + content = _openai_style_content(_build_prompt(sample), _images_for_sample(sample)) + resp = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": content}], + temperature=DEFAULT_TEMPERATURE, + reasoning_effort="none" if REASONING_MODE == "off" else "high", + ) + return ( + (resp.choices[0].message.content or "").strip(), + getattr(resp, "id", None), + _openai_usage(resp), + ) + + def call_gemini(sample: dict, model: str, client) -> tuple[str, str | None, dict]: from google.genai import types + prompt = _build_prompt(sample) m = model.lower() if REASONING_MODE == "off": @@ -463,7 +526,7 @@ def call_gemini(sample: dict, model: str, client) -> tuple[str, str | None, dict elif "pro" in m: thinking = types.ThinkingConfig(thinking_level="low") else: - thinking = types.ThinkingConfig(thinking_level="minimal") + thinking = types.ThinkingConfig(thinking_level=_gemini_floor(model)) else: if m.startswith("gemini-2.5"): thinking = types.ThinkingConfig(thinking_budget=-1) @@ -475,9 +538,9 @@ def call_gemini(sample: dict, model: str, client) -> tuple[str, str | None, dict ) contents: list = [] for img in _images_for_sample(sample): - contents.append(types.Part.from_bytes( - data=image_to_jpeg_bytes(img), mime_type="image/jpeg" - )) + contents.append( + types.Part.from_bytes(data=image_to_jpeg_bytes(img), mime_type="image/jpeg") + ) contents.append(prompt) resp = client.models.generate_content(model=model, contents=contents, config=config) return ( @@ -490,34 +553,55 @@ def call_gemini(sample: dict, model: str, client) -> tuple[str, str | None, dict def build_client(provider: str): if provider == "interfaze": from openai import OpenAI + api_key = os.getenv("INTERFAZE_API_KEY") if not api_key: raise RuntimeError("INTERFAZE_API_KEY missing") return OpenAI(base_url="https://api.interfaze.ai/v1", api_key=api_key) if provider == "openai": from openai import OpenAI + api_key = os.getenv("OPENAI_API_KEY") if not api_key: raise RuntimeError("OPENAI_API_KEY missing") return OpenAI(base_url="https://api.openai.com/v1", api_key=api_key) if provider == "anthropic": from anthropic import Anthropic + api_key = os.getenv("ANTHROPIC_API_KEY") if not api_key: raise RuntimeError("ANTHROPIC_API_KEY missing") return Anthropic(api_key=api_key) if provider == "gemini": from google import genai - api_key = os.getenv("GEMINI_KEY") or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY") + + api_key = ( + os.getenv("GEMINI_KEY") + or os.getenv("GEMINI_API_KEY") + or os.getenv("GOOGLE_API_KEY") + ) if not api_key: raise RuntimeError("GEMINI_KEY missing") return genai.Client(api_key=api_key) if provider == "openrouter": from openai import OpenAI - api_key = os.getenv("OPENROUTER_API_KEY") or os.getenv("OPENROUTER_KEY") + + api_key = ( + os.getenv("OPENROUTER_API_KEY") + or os.getenv("openrouter_api_key") + or os.getenv("OPENROUTER_KEY") + ) if not api_key: raise RuntimeError("OPENROUTER_API_KEY missing") return OpenAI(base_url="https://openrouter.ai/api/v1", api_key=api_key) + if provider == "fireworks": + from openai import OpenAI + + # .env in this repo may spell the key lowercase; accept either casing. + api_key = os.getenv("FIREWORKS_API_KEY") or os.getenv("fireworks_api_key") + if not api_key: + raise RuntimeError("FIREWORKS_API_KEY missing") + return OpenAI(base_url="https://api.fireworks.ai/inference/v1", api_key=api_key) raise ValueError(f"unknown provider: {provider}") @@ -528,6 +612,7 @@ def get_call_fn(provider: str): "anthropic": call_anthropic, "gemini": call_gemini, "openrouter": call_openrouter, + "fireworks": call_fireworks, }[provider] @@ -539,16 +624,26 @@ def model_slug(model: str) -> str: # Pipeline # --------------------------------------------------------------------------- -async def process_sample(sample: dict, call_fn, model: str, rate_limiter: RateLimiter, - writer: JsonlWriter, progress: dict, provider: str, - client) -> dict | None: + +async def process_sample( + sample: dict, + call_fn, + model: str, + rate_limiter: RateLimiter, + writer: JsonlWriter, + progress: dict, + provider: str, + client, +) -> dict | None: last_error: str | None = None for attempt in range(1, MAX_RETRIES + 1): await rate_limiter.acquire() start = time.perf_counter() try: - content, request_id, usage = await asyncio.to_thread(call_fn, sample, model, client) + content, request_id, usage = await asyncio.to_thread( + call_fn, sample, model, client + ) latency_ms = int((time.perf_counter() - start) * 1000) if not content: last_error = "empty response content" @@ -604,29 +699,48 @@ async def process_sample(sample: dict, call_fn, model: str, rate_limiter: RateLi await asyncio.sleep(2 ** (attempt - 1)) progress["failed"] += 1 - tqdm.write(f"[{provider}/{model} FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") + tqdm.write( + f"[{provider}/{model} FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}" + ) return None -async def run(provider: str, model: str, setting: str, pred_path: Path, limit: int | None): +async def run( + provider: str, model: str, setting: str, pred_path: Path, limit: int | None +): client = build_client(provider) call_fn = get_call_fn(provider) samples = load_samples(setting, limit) done_ids = load_completed_ids(pred_path) pending = [s for s in samples if s["id"] not in done_ids] - print(f"[{provider}/{model} {setting}] Total: {len(samples)}, " - f"resume: {len(done_ids)} done / {len(pending)} pending " - f"(checkpoint: {pred_path})") + print( + f"[{provider}/{model} {setting}] Total: {len(samples)}, " + f"resume: {len(done_ids)} done / {len(pending)} pending " + f"(checkpoint: {pred_path})" + ) if not pending: return writer = JsonlWriter(pred_path) rate_limiter = RateLimiter(RATE_LIMIT) - progress = {"total": len(pending), "done": 0, "correct": 0, "unparseable": 0, "failed": 0} + progress = { + "total": len(pending), + "done": 0, + "correct": 0, + "unparseable": 0, + "failed": 0, + } + + sem = asyncio.Semaphore(MAX_IN_FLIGHT) + + async def bounded(sample): + async with sem: + return await process_sample( + sample, call_fn, model, rate_limiter, writer, progress, provider, client + ) - tasks = [process_sample(s, call_fn, model, rate_limiter, writer, progress, provider, client) - for s in pending] + tasks = [bounded(s) for s in pending] try: await tqdm_asyncio.gather(*tasks, desc=f"{provider}/{model}/{setting}") except Exception: @@ -645,6 +759,7 @@ async def run(provider: str, model: str, setting: str, pred_path: Path, limit: i # Metrics # --------------------------------------------------------------------------- + def compute_metrics(results: list[dict]) -> dict: if not results: return {} @@ -672,7 +787,9 @@ def compute_metrics(results: list[dict]) -> dict: c = sum(1 for r in rows if r.get("correct")) per_difficulty[diff] = {"n": n, "accuracy": c / n if n else 0.0} - latencies = [r["latency_ms"] for r in results if isinstance(r.get("latency_ms"), int)] + latencies = [ + r["latency_ms"] for r in results if isinstance(r.get("latency_ms"), int) + ] latency_stats = {} if latencies: lats = sorted(latencies) @@ -698,18 +815,24 @@ def compute_metrics(results: list[dict]) -> dict: def print_summary(metrics: dict, setting: str, provider: str, model: str): print(f"\n{'=' * 68}") - print(f"MMMU Pro [{setting}] — {provider}/{model} (reasoning={REASONING_MODE}, temp={DEFAULT_TEMPERATURE})") + print( + f"MMMU Pro [{setting}] — {provider}/{model} (reasoning={REASONING_MODE}, temp={DEFAULT_TEMPERATURE})" + ) print(f"{'=' * 68}") print(f"Samples : {metrics['num_samples']}") print(f"Accuracy : {metrics['accuracy']:.4f}") print(f"Unparseable : {metrics['unparseable']}") if metrics.get("latency"): lat = metrics["latency"] - print(f"Latency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " - f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms") + print( + f"Latency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " + f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms" + ) -def run_evaluation(pred_path: Path, metrics_path: Path, provider: str, model: str, setting: str): +def run_evaluation( + pred_path: Path, metrics_path: Path, provider: str, model: str, setting: str +): if not pred_path.exists(): print(f"No predictions found at {pred_path}") return @@ -746,13 +869,24 @@ def run_evaluation(pred_path: Path, metrics_path: Path, provider: str, model: st def main(): global REASONING_MODE parser = argparse.ArgumentParser(description="MMMU Pro multi-provider runner") - parser.add_argument("--provider", required=True, - choices=["interfaze", "openai", "anthropic", "gemini", "openrouter"]) + parser.add_argument( + "--provider", + required=True, + choices=[ + "interfaze", + "openai", + "anthropic", + "gemini", + "openrouter", + "fireworks", + ], + ) parser.add_argument("--model", required=True) parser.add_argument("--setting", required=True, choices=list(SETTINGS.keys())) parser.add_argument("--reasoning", default="off", choices=["off", "high"]) - parser.add_argument("--limit", type=int, default=None, - help="Cap samples (smoke test)") + parser.add_argument( + "--limit", type=int, default=None, help="Cap samples (smoke test)" + ) parser.add_argument("--predict-only", action="store_true") parser.add_argument("--evaluate-only", action="store_true") args = parser.parse_args() @@ -765,9 +899,13 @@ def main(): if args.evaluate_only: run_evaluation(pred_path, metrics_path, args.provider, args.model, args.setting) elif args.predict_only: - asyncio.run(run(args.provider, args.model, args.setting, pred_path, limit=args.limit)) + asyncio.run( + run(args.provider, args.model, args.setting, pred_path, limit=args.limit) + ) else: - asyncio.run(run(args.provider, args.model, args.setting, pred_path, limit=args.limit)) + asyncio.run( + run(args.provider, args.model, args.setting, pred_path, limit=args.limit) + ) run_evaluation(pred_path, metrics_path, args.provider, args.model, args.setting) diff --git a/benchmarks/obj_detection/reeval_format_tolerant.py b/benchmarks/obj_detection/reeval_format_tolerant.py index c928a44..8996cf3 100644 --- a/benchmarks/obj_detection/reeval_format_tolerant.py +++ b/benchmarks/obj_detection/reeval_format_tolerant.py @@ -34,17 +34,18 @@ sys.path.insert(0, str(PROJECT_ROOT)) from benchmarks.obj_detection.refcoco import ( # noqa: E402 - compute_iou, compute_metrics, + compute_iou, + compute_metrics, ) RESULTS_DIR = PROJECT_ROOT / "results" TUPLE_PATTERN = re.compile( - r'\[\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*' - r'(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\]' + r"\[\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*" + r"(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\]" ) PAREN_PAIRS = re.compile( - r'\(\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\)\s*[^()]*?\s*' - r'\(\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\)' + r"\(\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\)\s*[^()]*?\s*" + r"\(\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\)" ) TLBR_JSON = re.compile( r'"top_left"\s*:\s*\{[^}]*?"x"\s*:\s*(-?\d+(?:\.\d+)?)[^}]*?"y"\s*:\s*' @@ -72,14 +73,33 @@ def extract_tuples(text: str) -> list[tuple[float, float, float, float]]: def all_interpretations(nums, w, h): """Yield (label, [x1,y1,x2,y2]) for every plausible interpretation of a 4-tuple under the canonical RefCOCO eval — covers xyxy/yxyx - order × {raw pixel, 0-1000 normalized, 0-1.0 float} scale.""" + order × {raw pixel, 2x-upscaled pixel, 0-1000 normalized, 0-1.0 float} + scale. + + The 2x-pixel family covers models that report coordinates in their own + internally-upscaled image space rather than the sent image's — Inkling + does this for images below its vision encoder's working resolution + (verified: a box at 50-75% of a 640x428 input comes back as + [500, 210, 750, 430], i.e. that region in 1280x856 space). Applied to + every model uniformly, like every other hypothesis here.""" n0, n1, n2, n3 = nums mx = max(abs(c) for c in nums) yield "pixel-xyxy", [n0, n1, n2, n3] yield "pixel-yxyx", [n1, n0, n3, n2] + # Only worth trying when the tuple overflows the image — otherwise it is + # just a shrunken duplicate of the pixel hypothesis. + if mx > min(w, h): + yield "pixel2x-xyxy", [n0 / 2, n1 / 2, n2 / 2, n3 / 2] + yield "pixel2x-yxyx", [n1 / 2, n0 / 2, n3 / 2, n2 / 2] if mx <= 1000: - yield "norm1000-xyxy", [n0 * w / 1000, n1 * h / 1000, n2 * w / 1000, n3 * h / 1000] - yield "norm1000-yxyx", [n1 * w / 1000, n0 * h / 1000, n3 * w / 1000, n2 * h / 1000] + yield ( + "norm1000-xyxy", + [n0 * w / 1000, n1 * h / 1000, n2 * w / 1000, n3 * h / 1000], + ) + yield ( + "norm1000-yxyx", + [n1 * w / 1000, n0 * h / 1000, n3 * w / 1000, n2 * h / 1000], + ) if mx <= 1.0: yield "norm1-xyxy", [n0 * w, n1 * h, n2 * w, n3 * h] yield "norm1-yxyx", [n1 * w, n0 * h, n3 * w, n2 * h] @@ -133,13 +153,35 @@ def write_records(records: list[dict], path: Path) -> None: f.write(json.dumps(r, ensure_ascii=False) + "\n") +def discover_targets() -> list[tuple[str, str]]: + """Every RefCOCO response file in results/ — identified by the grounding + fields rather than a filename convention, since the per-provider runners + tag their outputs differently. Skips this script's own oracle outputs.""" + targets = [] + for path in sorted(RESULTS_DIR.glob("*_responses.jsonl")): + if "_oracle_" in path.name: + continue + try: + with open(path) as f: + first = json.loads(f.readline() or "{}") + except (json.JSONDecodeError, OSError): + continue + if "gt_bbox_xyxy" not in first: + continue + tag = path.name[: -len("_responses.jsonl")] + label = first.get("model") or first.get("provider") or tag + targets.append((label, tag)) + return targets + + def main(): - targets = [ - ("interfaze", "refcoco_testA"), - ("gpt-5.5", "gpt55_refcoco_testA"), - ("kimi-k2.6", "kimi_k26_refcoco_testA"), - ] - print(f"{'model':<14} {'orig_acc':>10} {'oracle_acc':>11} {'mean_iou':>10} {'changed':>9}") + targets = discover_targets() + if not targets: + print(f"No RefCOCO response files found in {RESULTS_DIR}") + return + print( + f"{'model':<14} {'orig_acc':>10} {'oracle_acc':>11} {'mean_iou':>10} {'changed':>9}" + ) print("-" * 60) for label, tag in targets: src = RESULTS_DIR / f"{tag}_responses.jsonl" @@ -164,10 +206,14 @@ def main(): with open(out_metrics, "w") as f: json.dump(metrics, f, indent=2) - print(f"{label:<14} {orig_correct/orig_total:>10.4f} " - f"{metrics['accuracy']:>11.4f} {metrics['mean_iou']:>10.4f} " - f"{info['n_changed']:>9d}") - print(f" interpretations used: {dict(sorted(info['label_counts'].items(), key=lambda x: -x[1]))}") + print( + f"{label:<14} {orig_correct / orig_total:>10.4f} " + f"{metrics['accuracy']:>11.4f} {metrics['mean_iou']:>10.4f} " + f"{info['n_changed']:>9d}" + ) + print( + f" interpretations used: {dict(sorted(info['label_counts'].items(), key=lambda x: -x[1]))}" + ) if __name__ == "__main__": diff --git a/benchmarks/obj_detection/refcoco_multi.py b/benchmarks/obj_detection/refcoco_multi.py index 9239abe..f89f845 100644 --- a/benchmarks/obj_detection/refcoco_multi.py +++ b/benchmarks/obj_detection/refcoco_multi.py @@ -17,6 +17,7 @@ Keys are loaded from ~/interfaze/.env.local (OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_KEY). """ +import os import sys import json import time @@ -28,23 +29,35 @@ from pathlib import Path from datasets import load_dataset +from dotenv import load_dotenv from tqdm import tqdm from tqdm.asyncio import tqdm_asyncio +load_dotenv() + PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent sys.path.insert(0, str(PROJECT_ROOT)) # Reuse parsing/IoU from the interfaze script — guarantees identical scoring. from benchmarks.obj_detection.refcoco import ( # noqa: E402 - parse_box, compute_iou, coco_bbox_to_xyxy, - JsonlWriter, build_samples, load_completed_ids, load_records, - compute_metrics, print_summary, PROMPT_TEMPLATE, IOU_THRESHOLD, + parse_box, + compute_iou, + coco_bbox_to_xyxy, + JsonlWriter, + build_samples, + load_completed_ids, + load_records, + compute_metrics, + print_summary, + PROMPT_TEMPLATE, + IOU_THRESHOLD, ) class RateLimiter: """Simple async token-bucket. Local to this script — refcoco.py does not export one, so we don't try to import it.""" + def __init__(self, rate: int): self.rate = rate self.tokens = rate @@ -63,18 +76,46 @@ async def acquire(self): return await asyncio.sleep(1 / self.rate) + RESULTS_DIR = PROJECT_ROOT / "results" DEFAULT_DATASET = "lmms-lab/RefCOCO" DEFAULT_SPLIT = "testA" RATE_LIMIT = 25 MAX_RETRIES = 3 +# Cap on requests in flight. RATE_LIMIT bounds how many requests *start* per +# second but not how many are outstanding: gather() launches every pending +# sample at once, so hundreds pile up, queue server-side, and eventually 429 +# after tens of minutes of latency. Override with BENCH_MAX_IN_FLIGHT. +MAX_IN_FLIGHT = int(os.getenv("BENCH_MAX_IN_FLIGHT", "8")) + +# reasoning.effort sent to OpenRouter-hosted models. Override per run. +OPENROUTER_EFFORT = os.getenv("BENCH_OPENROUTER_EFFORT", "none") + +def _gemini_floor(model: str) -> str: + """Lowest thinking_level the given Gemini model actually accepts. + + gemini-3.7+ flash rejects MINIMAL server-side ("Thinking level MINIMAL is + not supported for this model"), and the SDK enum has no "off"/"none", so + "low" is its true floor. Older 3.x flash models still take "minimal". + BENCH_GEMINI_FLOOR overrides for anything not covered here. + """ + override = os.getenv("BENCH_GEMINI_FLOOR") + if override: + return override + m = model.lower() + if any(v in m for v in ("3.7", "3.8", "3.9", "4.")): + return "low" + return "minimal" + def _load_interfaze_env() -> dict: - """Parse ~/interfaze/.env.local and return a dict of keys.""" + """Parse ~/interfaze/.env.local and return a dict of keys. Optional — the + project .env (loaded above) is enough for providers keyed there, so a + missing file is not fatal.""" path = Path.home() / "interfaze" / ".env.local" if not path.exists(): - raise FileNotFoundError(f"Expected keys at {path}") + return {} env = {} for line in path.read_text().splitlines(): line = line.strip() @@ -92,6 +133,7 @@ def _load_interfaze_env() -> dict: # Each adapter is a function: (image_pil, prompt, model) -> (content, request_id) # Runs synchronously inside asyncio.to_thread. + def _image_to_jpeg_bytes(image, max_side: int = 1024) -> tuple[bytes, int, int]: if image.mode != "RGB": image = image.convert("RGB") @@ -118,13 +160,15 @@ def call_openai(image, prompt: str, model: str, client) -> tuple[str, str, int, data_url = f"data:image/jpeg;base64,{b64}" kwargs = { "model": model, - "messages": [{ - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - {"type": "image_url", "image_url": {"url": data_url}}, - ], - }], + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + {"type": "image_url", "image_url": {"url": data_url}}, + ], + } + ], } if model.startswith("gpt-5") or model.startswith("o"): kwargs["reasoning_effort"] = "none" @@ -142,19 +186,87 @@ def call_anthropic(image, prompt: str, model: str, client) -> tuple[str, str, in model=model, max_tokens=1024, thinking={"type": "disabled"}, - messages=[{ - "role": "user", - "content": [ - {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": b64}}, - {"type": "text", "text": prompt}, - ], - }], + messages=[ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": b64, + }, + }, + {"type": "text", "text": prompt}, + ], + } + ], ) parts = [b.text for b in resp.content if getattr(b, "type", None) == "text"] content = "\n".join(parts).strip() return content, resp.id, w, h +def call_openrouter(image, prompt: str, model: str, client) -> tuple[str, str, int, int]: + """OpenRouter-hosted VLMs. Reasoning is routed through extra_body, which is + OpenRouter's shape rather than the OpenAI `reasoning_effort` field. + + NOTE for thinkingmachines/*: on OpenRouter effort="none" genuinely disables + thinking (0 reasoning tokens), whereas the same value on Fireworks is only a + floor and still emits hundreds. The two hosts are not interchangeable at the + same nominal setting — see README. + """ + img_bytes, w, h = _image_to_jpeg_bytes(image) + b64 = base64.b64encode(img_bytes).decode("utf-8") + resp = client.chat.completions.create( + model=model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{b64}"}, + }, + ], + } + ], + temperature=0.0, + extra_body={"reasoning": {"effort": OPENROUTER_EFFORT}}, + ) + content = (resp.choices[0].message.content or "").strip() + return content, getattr(resp, "id", "") or "", w, h + + +def call_fireworks(image, prompt: str, model: str, client) -> tuple[str, str, int, int]: + """Fireworks-hosted models (Inkling). Inkling always thinks; "none" is the + reasoning floor, which is the closest match to the interfaze run's + reasoning_effort=None.""" + img_bytes, w, h = _image_to_jpeg_bytes(image) + b64 = base64.b64encode(img_bytes).decode("utf-8") + resp = client.chat.completions.create( + model=model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{b64}"}, + }, + ], + } + ], + temperature=0.0, + reasoning_effort="none", + ) + content = (resp.choices[0].message.content or "").strip() + return content, getattr(resp, "id", "") or "", w, h + + def call_gemini(image, prompt: str, model: str, client) -> tuple[str, str, int, int]: """Gemini 3.x has thinking ON by default (HIGH for Pro, can't disable). We always override to the LOWEST supported level for the chosen model: @@ -162,12 +274,13 @@ def call_gemini(image, prompt: str, model: str, client) -> tuple[str, str, int, gemini-3*-flash*: 'minimal' (Flash supports 'minimal' as the floor) Gemini 2.5 used thinking_budget; we don't support that path here.""" from google.genai import types + img_bytes, w, h = _image_to_jpeg_bytes(image) m = model.lower() if "pro" in m: thinking_level = "low" else: - thinking_level = "minimal" + thinking_level = _gemini_floor(model) config = types.GenerateContentConfig( temperature=0.0, thinking_config=types.ThinkingConfig(thinking_level=thinking_level), @@ -189,9 +302,17 @@ def call_gemini(image, prompt: str, model: str, client) -> tuple[str, str, int, # Provider-agnostic pipeline (mirrors refcoco.py's process_sample) # -------------------------------------------------------------------------- -async def process_sample(sample: dict, call_fn, model: str, rate_limiter, - writer: JsonlWriter, progress: dict, provider: str, - client) -> dict | None: + +async def process_sample( + sample: dict, + call_fn, + model: str, + rate_limiter, + writer: JsonlWriter, + progress: dict, + provider: str, + client, +) -> dict | None: """Pre-resize the image to know the exact dims that will be sent, embed those dims in the prompt, then call the provider adapter.""" tmp_bytes, sent_w, sent_h = _image_to_jpeg_bytes(sample["image"]) @@ -251,7 +372,7 @@ async def process_sample(sample: dict, call_fn, model: str, rate_limiter, tqdm.write( f"[{provider} {progress['done']}/{progress['total']}] {mark} " f"id={sample['id']} iou={iou:.3f} " - f"pred={pred_box} gt={[round(x,1) for x in gt_xyxy]} " + f"pred={pred_box} gt={[round(x, 1) for x in gt_xyxy]} " f"latency={latency_ms}ms req_id={request_id} attempt={attempt}" ) return record @@ -267,7 +388,9 @@ async def process_sample(sample: dict, call_fn, model: str, rate_limiter, await asyncio.sleep(2 ** (attempt - 1)) progress["failed"] += 1 - tqdm.write(f"[{provider} FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") + tqdm.write( + f"[{provider} FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}" + ) return None @@ -275,6 +398,7 @@ async def process_sample(sample: dict, call_fn, model: str, rate_limiter, # CLI # -------------------------------------------------------------------------- + def build_tag(dataset: str, split: str, provider: str, model: str) -> str: ds_slug = dataset.split("/")[-1].lower().replace("+", "plus") model_slug = model.replace("/", "_").replace(":", "_") @@ -284,13 +408,47 @@ def build_tag(dataset: str, split: str, provider: str, model: str) -> str: def build_client(provider: str, env: dict): if provider == "openai": from openai import OpenAI + return OpenAI(api_key=env["OPENAI_API_KEY"]) if provider == "anthropic": from anthropic import Anthropic + return Anthropic(api_key=env["ANTHROPIC_API_KEY"]) if provider == "gemini": from google import genai - return genai.Client(api_key=env["GEMINI_KEY"]) + + api_key = ( + env.get("GEMINI_KEY") + or os.getenv("GEMINI_API_KEY") + or os.getenv("GEMINI_KEY") + or os.getenv("GOOGLE_API_KEY") + ) + if not api_key: + raise RuntimeError("GEMINI_API_KEY missing from .env") + return genai.Client(api_key=api_key) + if provider == "openrouter": + from openai import OpenAI + + api_key = ( + env.get("OPENROUTER_API_KEY") + or os.getenv("OPENROUTER_API_KEY") + or os.getenv("openrouter_api_key") + ) + if not api_key: + raise RuntimeError("OPENROUTER_API_KEY missing from .env") + return OpenAI(base_url="https://openrouter.ai/api/v1", api_key=api_key) + if provider == "fireworks": + from openai import OpenAI + + # .env in this repo may spell the key lowercase; accept either casing. + api_key = ( + env.get("FIREWORKS_API_KEY") + or os.getenv("FIREWORKS_API_KEY") + or os.getenv("fireworks_api_key") + ) + if not api_key: + raise RuntimeError("FIREWORKS_API_KEY missing from .env") + return OpenAI(base_url="https://api.fireworks.ai/inference/v1", api_key=api_key) raise ValueError(f"unknown provider: {provider}") @@ -299,11 +457,19 @@ def get_call_fn(provider: str): "openai": call_openai, "anthropic": call_anthropic, "gemini": call_gemini, + "fireworks": call_fireworks, + "openrouter": call_openrouter, }[provider] -async def run(provider: str, model: str, dataset_name: str, split: str, - pred_path: Path, limit: int | None): +async def run( + provider: str, + model: str, + dataset_name: str, + split: str, + pred_path: Path, + limit: int | None, +): env = _load_interfaze_env() client = build_client(provider, env) call_fn = get_call_fn(provider) @@ -317,26 +483,43 @@ async def run(provider: str, model: str, dataset_name: str, split: str, if limit is not None: pending = pending[:limit] print(f"--limit applied: will run at most {limit} sample(s)") - print(f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})") + print( + f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " + f"(checkpoint: {pred_path})" + ) if not pending: return writer = JsonlWriter(pred_path) rate_limiter = RateLimiter(RATE_LIMIT) progress = {"total": len(pending), "done": 0, "correct": 0, "failed": 0} - tasks = [process_sample(s, call_fn, model, rate_limiter, writer, progress, provider, client) - for s in pending] + sem = asyncio.Semaphore(MAX_IN_FLIGHT) + + async def bounded(sample): + async with sem: + return await process_sample( + sample, call_fn, model, rate_limiter, writer, progress, provider, client + ) + + tasks = [bounded(s) for s in pending] try: await tqdm_asyncio.gather(*tasks, desc=f"{provider}/{model}") except Exception: traceback.print_exc() - print(f"\n[{provider}/{model}] Run finished: {progress['done']}/{progress['total']} answered, " - f"{progress['correct']} correct, {progress['failed']} failed.") + print( + f"\n[{provider}/{model}] Run finished: {progress['done']}/{progress['total']} answered, " + f"{progress['correct']} correct, {progress['failed']} failed." + ) -def run_evaluation(dataset_name: str, split: str, pred_path: Path, metrics_path: Path, - provider: str, model: str): +def run_evaluation( + dataset_name: str, + split: str, + pred_path: Path, + metrics_path: Path, + provider: str, + model: str, +): if not pred_path.exists(): print(f"No predictions found at {pred_path}") sys.exit(1) @@ -348,9 +531,12 @@ def run_evaluation(dataset_name: str, split: str, pred_path: Path, metrics_path: print_summary(metrics, dataset_name, split) output = { **metrics, - "dataset": dataset_name, "split": split, - "provider": provider, "model": model, - "rate_limit": RATE_LIMIT, "iou_threshold": IOU_THRESHOLD, + "dataset": dataset_name, + "split": split, + "provider": provider, + "model": model, + "rate_limit": RATE_LIMIT, + "iou_threshold": IOU_THRESHOLD, } metrics_path.parent.mkdir(parents=True, exist_ok=True) with open(metrics_path, "w") as f: @@ -360,7 +546,11 @@ def run_evaluation(dataset_name: str, split: str, pred_path: Path, metrics_path: def main(): parser = argparse.ArgumentParser(description="Multi-provider RefCOCO eval") - parser.add_argument("--provider", required=True, choices=["openai", "anthropic", "gemini"]) + parser.add_argument( + "--provider", + required=True, + choices=["openai", "anthropic", "gemini", "fireworks", "openrouter"], + ) parser.add_argument("--model", required=True, help="Provider-specific model id") parser.add_argument("--dataset", default=DEFAULT_DATASET) parser.add_argument("--split", default=DEFAULT_SPLIT) @@ -374,16 +564,34 @@ def main(): metrics_path = RESULTS_DIR / f"{tag}_metrics.json" if args.evaluate_only: - run_evaluation(args.dataset, args.split, pred_path, metrics_path, - args.provider, args.model) + run_evaluation( + args.dataset, args.split, pred_path, metrics_path, args.provider, args.model + ) elif args.predict_only: - asyncio.run(run(args.provider, args.model, args.dataset, args.split, - pred_path, limit=args.limit)) + asyncio.run( + run( + args.provider, + args.model, + args.dataset, + args.split, + pred_path, + limit=args.limit, + ) + ) else: - asyncio.run(run(args.provider, args.model, args.dataset, args.split, - pred_path, limit=args.limit)) - run_evaluation(args.dataset, args.split, pred_path, metrics_path, - args.provider, args.model) + asyncio.run( + run( + args.provider, + args.model, + args.dataset, + args.split, + pred_path, + limit=args.limit, + ) + ) + run_evaluation( + args.dataset, args.split, pred_path, metrics_path, args.provider, args.model + ) if __name__ == "__main__": diff --git a/benchmarks/ocrbench_v2/ocrbench_v2_fireworks.py b/benchmarks/ocrbench_v2/ocrbench_v2_fireworks.py new file mode 100644 index 0000000..e63d37d --- /dev/null +++ b/benchmarks/ocrbench_v2/ocrbench_v2_fireworks.py @@ -0,0 +1,441 @@ +""" +OCRBench v2 benchmark for Fireworks-hosted models (default: Inkling). +10,000 QA pairs across 30 task types (EN + CN). + +Inkling always thinks; reasoning_effort="none" is its floor, not a true off. + +Usage: + uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks + uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks --model accounts/fireworks/models/inkling-small + uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks --predict-only + uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks --evaluate-only +""" + +import os +import re +import sys +import json +import asyncio +import argparse +import base64 +from pathlib import Path +from io import BytesIO + +from datasets import load_dataset +from tqdm.asyncio import tqdm_asyncio + +PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent +BENCHMARK_DIR = Path(__file__).resolve().parent +RESULTS_DIR = PROJECT_ROOT / "results" + +sys.path.insert(0, str(PROJECT_ROOT)) +from src.commons_fireworks import INKLING, fireworks_client # noqa: E402 + +# Provider is selectable because the same model can be reachable on more than one +# host, and the hosts are NOT equivalent: on OpenRouter reasoning effort "none" +# truly disables thinking, while on Fireworks it is only a floor. +PROVIDER = "fireworks" +CLIENT = fireworks_client + + +def set_provider(provider: str) -> None: + global PROVIDER, CLIENT + PROVIDER = provider + if provider == "openrouter": + from src.commons_openrouter import openrouter_client + + CLIENT = openrouter_client + else: + CLIENT = fireworks_client + + +def _reasoning_kwargs() -> dict: + if PROVIDER == "openrouter": + return {"extra_body": {"reasoning": {"effort": REASONING_EFFORT}}} + return {"reasoning_effort": REASONING_EFFORT} + +MODEL = INKLING +REASONING_EFFORT = "none" +RATE_LIMIT = 25 +MAX_RETRIES = 3 + +# Cap on requests in flight. RATE_LIMIT bounds how many requests *start* per +# second but not how many are outstanding — a whole batch launched at once +# queues server-side and eventually 429s after minutes of latency. Override +# with BENCH_MAX_IN_FLIGHT. +MAX_IN_FLIGHT = int(os.getenv("BENCH_MAX_IN_FLIGHT", "8")) + +# Set from MODEL in main() so inkling and inkling-small keep separate +# checkpoints and can run side by side. +TAG = "ocrbench_v2_fireworks_inkling" +PRED_OUTPUT = RESULTS_DIR / f"{TAG}_predictions.json" +EVAL_OUTPUT = RESULTS_DIR / f"{TAG}_scored.json" +METRICS_OUTPUT = RESULTS_DIR / f"{TAG}_metrics.json" + +TEXT_SPOTTING_PROMPT_TEMPLATE = """Use OCR on this image to spot all text at {level}. The OCR tool returns each detected text region with its text content and four corner coordinates: top_left, top_right, bottom_left, bottom_right (each as an x,y pixel pair). + +Then use run code to write a Python script that takes those OCR results and: +1. For each text region, compute the axis-aligned bounding box from the four corners: + - x1 = min of all x coordinates (leftmost) + - y1 = min of all y coordinates (topmost) + - x2 = max of all x coordinates (rightmost) + - y2 = max of all y coordinates (bottommost) +2. Normalize each coordinate to the range 0-1000 by dividing by the image width (for x) or height (for y) and multiplying by 1000, then rounding to an integer. +3. Print the results as a Python list. + +Your final answer must be ONLY a Python list in this exact format, with no markdown, no code fences, no explanation: +[(x1, y1, x2, y2, "text"), (x1, y1, x2, y2, "text"), ...]""" + + +def model_slug(model: str) -> str: + """accounts/fireworks/models/inkling-small -> inkling-small""" + return re.sub(r"[^a-z0-9]+", "-", model.rsplit("/", 1)[-1].lower()).strip("-") + + +def set_paths(model: str) -> None: + global TAG, PRED_OUTPUT, EVAL_OUTPUT, METRICS_OUTPUT + TAG = f"ocrbench_v2_fireworks_{model_slug(model)}" + PRED_OUTPUT = RESULTS_DIR / f"{TAG}_predictions.json" + EVAL_OUTPUT = RESULTS_DIR / f"{TAG}_scored.json" + METRICS_OUTPUT = RESULTS_DIR / f"{TAG}_metrics.json" + + +def get_spotting_prompt(original_question: str) -> str: + if "line-level" in original_question: + return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="line-level") + return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="word-level") + + +class RateLimiter: + def __init__(self, rate: int): + self.rate = rate + self.tokens = rate + self.last_refill = 0.0 + self._lock = asyncio.Lock() + + async def acquire(self): + while True: + async with self._lock: + now = asyncio.get_running_loop().time() + elapsed = now - self.last_refill + self.tokens = min(self.rate, self.tokens + elapsed * self.rate) + self.last_refill = now + if self.tokens >= 1: + self.tokens -= 1 + return + await asyncio.sleep(1 / self.rate) + + +def pil_to_data_url(image) -> str: + buffer = BytesIO() + if image.mode != "RGB": + image = image.convert("RGB") + image.save(buffer, format="JPEG", quality=95) + b64 = base64.b64encode(buffer.getvalue()).decode("utf-8") + return f"data:image/jpeg;base64,{b64}" + + +def build_messages(question: str, image_url: str) -> list[dict]: + return [ + { + "role": "user", + "content": [ + {"type": "text", "text": question}, + {"type": "image_url", "image_url": {"url": image_url}}, + ], + } + ] + + +async def process_sample(sample_meta: dict, rate_limiter, sem: asyncio.Semaphore): + async with sem: + return await _process_sample(sample_meta, rate_limiter) + + +async def _process_sample(sample_meta: dict, rate_limiter): + messages = build_messages(sample_meta["question"], sample_meta["image_url"]) + + for attempt in range(MAX_RETRIES): + await rate_limiter.acquire() + try: + response = await asyncio.to_thread( + CLIENT.chat.completions.create, + model=MODEL, + messages=messages, + **_reasoning_kwargs(), + ) + return response.choices[0].message.content or "" + except Exception as e: + if attempt < MAX_RETRIES - 1: + await asyncio.sleep(2**attempt) + else: + print( + f"Failed after {MAX_RETRIES} attempts for id={sample_meta['id']}: {e}" + ) + return "" + + +BATCH_SIZE = 200 + + +async def run_predictions(limit: int | None = None): + print("Loading OCRBench v2 from HuggingFace...") + dataset = load_dataset("lmms-lab/OCRBench-v2", split="test") + if limit is not None: + dataset = dataset.select(range(min(limit, len(dataset)))) + print(f"--limit applied: first {len(dataset)} sample(s) only") + total = len(dataset) + print(f"Loaded {total} samples") + + RESULTS_DIR.mkdir(parents=True, exist_ok=True) + + existing = {} + if PRED_OUTPUT.exists(): + with open(PRED_OUTPUT) as f: + for item in json.load(f): + if item.get("predict", "") != "": + existing[item["id"]] = item + print(f"Resuming: {len(existing)} successful predictions found, skipping them") + + rate_limiter = RateLimiter(RATE_LIMIT) + sem = asyncio.Semaphore(MAX_IN_FLIGHT) + output_data = {} + output_data.update(existing) + num_retried = 0 + + from tqdm import tqdm + + for batch_start in tqdm(range(0, total, BATCH_SIZE), desc="Batches"): + batch_end = min(batch_start + BATCH_SIZE, total) + batch = dataset[batch_start:batch_end] + + samples = [] + for i in range(len(batch["id"])): + sample_id = batch["id"][i] + if sample_id in existing: + continue + image_url = pil_to_data_url(batch["image"][i]) + question = batch["question"][i] + if batch["type"][i] == "text spotting en": + question = get_spotting_prompt(question) + samples.append( + { + "id": sample_id, + "dataset_name": batch["dataset_name"][i], + "type": batch["type"][i], + "question": question, + "answers": batch["answers"][i], + "image_url": image_url, + } + ) + del batch + + if not samples: + continue + + num_retried += len(samples) + tasks = [process_sample(s, rate_limiter, sem) for s in samples] + predictions = await tqdm_asyncio.gather( + *tasks, desc=f"Predicting {batch_start}-{batch_end}", leave=False + ) + + for sample, pred in zip(samples, predictions): + output_data[sample["id"]] = { + "id": sample["id"], + "dataset_name": sample["dataset_name"], + "type": sample["type"], + "question": sample["question"], + "answers": sample["answers"], + "predict": pred, + } + + del samples, predictions + + # Checkpoint after every batch — a 10k-sample run is too long to lose. + final_data = [output_data[i] for i in sorted(output_data.keys())] + with open(PRED_OUTPUT, "w", encoding="utf-8") as f: + json.dump(final_data, f, ensure_ascii=False, indent=2) + + final_data = [output_data[i] for i in sorted(output_data.keys())] + with open(PRED_OUTPUT, "w", encoding="utf-8") as f: + json.dump(final_data, f, ensure_ascii=False, indent=2) + + num_failures = sum(1 for d in final_data if d.get("predict", "") == "") + print(f"\nPredictions saved to {PRED_OUTPUT}") + print( + f"Total: {total} | Retried: {num_retried} | Remaining failures: {num_failures}" + ) + + +def run_evaluation(): + if not PRED_OUTPUT.exists(): + print(f"No predictions found at {PRED_OUTPUT}") + print("Run with --predict-only first, or without flags to do both.") + sys.exit(1) + + eval_scripts_dir = BENCHMARK_DIR / "eval_scripts" + sys.path.insert(0, str(eval_scripts_dir)) + + import os + + original_cwd = os.getcwd() + os.chdir(BENCHMARK_DIR) + + print("Step 1: Scoring individual samples...") + from benchmarks.ocrbench_v2.eval_scripts.eval import process_predictions # noqa: E402 + + EVAL_OUTPUT.parent.mkdir(parents=True, exist_ok=True) + process_predictions(str(PRED_OUTPUT), str(EVAL_OUTPUT)) + + os.chdir(original_cwd) + print(f"Scored results saved to {EVAL_OUTPUT}") + + print("\nStep 2: Computing overall metrics...") + with open(EVAL_OUTPUT) as f: + scored_data = json.load(f) + + en_scores = { + "text_recognition": [], + "text_detection": [], + "text_spotting": [], + "relationship_extraction": [], + "element_parsing": [], + "mathematical_calculation": [], + "visual_text_understanding": [], + "knowledge_reasoning": [], + } + cn_scores = { + "text_recognition": [], + "relationship_extraction": [], + "element_parsing": [], + "visual_text_understanding": [], + "knowledge_reasoning": [], + } + + type_to_en = { + "text recognition en": "text_recognition", + "fine-grained text recognition en": "text_recognition", + "full-page OCR en": "text_recognition", + "text grounding en": "text_detection", + "VQA with position en": "text_detection", + "text spotting en": "text_spotting", + "key information extraction en": "relationship_extraction", + "key information mapping en": "relationship_extraction", + "document parsing en": "element_parsing", + "chart parsing en": "element_parsing", + "table parsing en": "element_parsing", + "formula recognition en": "element_parsing", + "math QA en": "mathematical_calculation", + "text counting en": "mathematical_calculation", + "document classification en": "visual_text_understanding", + "cognition VQA en": "visual_text_understanding", + "diagram QA en": "visual_text_understanding", + "reasoning VQA en": "knowledge_reasoning", + "science QA en": "knowledge_reasoning", + "APP agent en": "knowledge_reasoning", + "ASCII art classification en": "knowledge_reasoning", + } + type_to_cn = { + "full-page OCR cn": "text_recognition", + "key information extraction cn": "relationship_extraction", + "handwritten answer extraction cn": "relationship_extraction", + "document parsing cn": "element_parsing", + "table parsing cn": "element_parsing", + "formula recognition cn": "element_parsing", + "cognition VQA cn": "visual_text_understanding", + "reasoning VQA cn": "knowledge_reasoning", + "text translation cn": "knowledge_reasoning", + } + + for item in scored_data: + if "ignore" in item: + continue + t = item["type"] + if t in type_to_en: + en_scores[type_to_en[t]].append(item["score"]) + elif t in type_to_cn: + cn_scores[type_to_cn[t]].append(item["score"]) + + def avg(lst): + return sum(lst) / len(lst) if lst else 0.0 + + en_avgs = {k: avg(v) for k, v in en_scores.items() if v} + cn_avgs = {k: avg(v) for k, v in cn_scores.items() if v} + en_overall = avg(list(en_avgs.values())) + cn_overall = avg(list(cn_avgs.values())) + + print(f"\n{'=' * 60}") + print(f"OCRBench v2 Results ({MODEL})") + print(f"{'=' * 60}") + print(f"\n{'Category':<30} {'EN':>8} {'CN':>8}") + print("-" * 48) + all_cats = sorted(set(list(en_scores.keys()) + list(cn_scores.keys()))) + for cat in all_cats: + en_val = f"{en_avgs[cat]:.3f}" if cat in en_avgs else " -" + cn_val = f"{cn_avgs[cat]:.3f}" if cat in cn_avgs else " -" + print(f"{cat:<30} {en_val:>8} {cn_val:>8}") + print("-" * 48) + print(f"{'OVERALL':<30} {en_overall:>8.3f} {cn_overall:>8.3f}") + + metrics = { + "en_scores": { + k: {"avg": avg(v), "count": len(v)} for k, v in en_scores.items() + }, + "cn_scores": { + k: {"avg": avg(v), "count": len(v)} for k, v in cn_scores.items() + }, + "en_overall": en_overall, + "cn_overall": cn_overall, + "provider": "fireworks", + "model": MODEL, + "reasoning_effort": REASONING_EFFORT, + } + with open(METRICS_OUTPUT, "w") as f: + json.dump(metrics, f, indent=2) + print(f"\nMetrics saved to {METRICS_OUTPUT}") + + +def main(): + global MODEL, REASONING_EFFORT + parser = argparse.ArgumentParser( + description="OCRBench v2 benchmark for Fireworks-hosted models (Inkling)" + ) + parser.add_argument( + "--model", default=INKLING, help="Model id for the chosen provider" + ) + parser.add_argument( + "--provider", default="fireworks", choices=["fireworks", "openrouter"] + ) + parser.add_argument( + "--reasoning", default="none", help="reasoning effort passed to the provider" + ) + parser.add_argument( + "--predict-only", action="store_true", help="Only generate predictions" + ) + parser.add_argument( + "--evaluate-only", action="store_true", help="Only run evaluation" + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Only predict the first N dataset rows (smoke test)", + ) + args = parser.parse_args() + + MODEL = args.model + REASONING_EFFORT = args.reasoning + set_provider(args.provider) + set_paths(MODEL) + + if args.evaluate_only: + run_evaluation() + elif args.predict_only: + asyncio.run(run_predictions(limit=args.limit)) + else: + asyncio.run(run_predictions(limit=args.limit)) + run_evaluation() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/ocrbench_v2/ocrbench_v2_gemini.py b/benchmarks/ocrbench_v2/ocrbench_v2_gemini.py index 224f49a..b862955 100644 --- a/benchmarks/ocrbench_v2/ocrbench_v2_gemini.py +++ b/benchmarks/ocrbench_v2/ocrbench_v2_gemini.py @@ -327,7 +327,11 @@ def main(): parser.add_argument( "--evaluate-only", action="store_true", help="Only run evaluation" ) + parser.add_argument( + "--model", default=MODEL, help="Gemini model id (e.g. gemini-3.7-flash)" + ) args = parser.parse_args() + globals()["MODEL"] = args.model if args.evaluate_only: run_evaluation() diff --git a/benchmarks/olmocr/bench/runners/run_fireworks.py b/benchmarks/olmocr/bench/runners/run_fireworks.py new file mode 100644 index 0000000..1cbbe87 --- /dev/null +++ b/benchmarks/olmocr/bench/runners/run_fireworks.py @@ -0,0 +1,99 @@ +import os + +from openai import OpenAI + +from benchmarks.olmocr.data.renderpdf import render_pdf_to_base64png + +_client = None + +INKLING = "accounts/fireworks/models/inkling" + + +def _get_client() -> OpenAI: + """Host is chosen by BENCH_OLMOCR_PROVIDER (fireworks|openrouter). The hosts + are not equivalent at the same nominal reasoning setting — see README.""" + global _client + if _client is None: + if os.getenv("BENCH_OLMOCR_PROVIDER") == "openrouter": + api_key = os.getenv("OPENROUTER_API_KEY") or os.getenv("openrouter_api_key") + if not api_key: + raise SystemExit("OPENROUTER_API_KEY not set") + _client = OpenAI( + api_key=api_key, base_url="https://openrouter.ai/api/v1" + ) + return _client + # .env in this repo may spell the key lowercase; accept either casing. + api_key = os.getenv("FIREWORKS_API_KEY") or os.getenv("fireworks_api_key") + if not api_key: + raise SystemExit( + "FIREWORKS_API_KEY not set — get it from " + "https://app.fireworks.ai/settings/users/api-keys" + ) + _client = OpenAI( + api_key=api_key, base_url="https://api.fireworks.ai/inference/v1" + ) + return _client + + +def run_fireworks( + pdf_path: str, + page_num: int = 1, + model: str = INKLING, + target_longest_image_dim: int = 2048, +) -> str: + """Convert a PDF page to markdown via a Fireworks-hosted model (Inkling). + + reasoning_effort='none' is Inkling's floor — it still thinks, so the page + transcription is preceded by reasoning tokens we pay for but discard.""" + image_base64 = render_pdf_to_base64png( + pdf_path, page_num=page_num, target_longest_image_dim=target_longest_image_dim + ) + + client = _get_client() + + prompt = ( + "Below is the image of one page of a PDF document. " + "Just return the plain text representation of this document as if you were reading it naturally.\n" + "Turn equations into LaTeX using \\( \\) for inline math and \\[ \\] for display math. " + "Never describe equations in words — always use LaTeX notation. " + "Turn tables into markdown format.\n" + "Remove the headers and footers completely — do not include any text " + "that appears at the very top or very bottom of the page outside the main body content. " + "This includes page numbers, journal names, author names in running headers, " + "copyright lines, DOI lines, citation requests, institutional addresses in margins, " + "and download dates. Keep references and footnotes that are part of the body.\n" + "For multi-column layouts, read each column top to bottom before moving to the next.\n" + "Read any natural handwriting.\n" + "This is likely one page out of several in the document, so be sure to preserve " + "any sentences that come from the previous page, or continue onto the next page, exactly as they are.\n" + "If there is no text at all that you think you should read, you can output null.\n" + "Do not hallucinate." + ) + + response = client.chat.completions.create( + model=model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{image_base64}"}, + }, + ], + } + ], + max_tokens=20000, + **( + {"extra_body": {"reasoning": {"effort": os.getenv("BENCH_OLMOCR_EFFORT", "none")}}} + if os.getenv("BENCH_OLMOCR_PROVIDER") == "openrouter" + else {"reasoning_effort": os.getenv("BENCH_OLMOCR_EFFORT", "none")} + ), + ) + + assert len(response.choices) > 0 + raw = response.choices[0].message.content + if raw is None or raw.strip().lower() in ("null", "none", "n/a", ""): + return "" + return raw diff --git a/benchmarks/olmocr/bench/runners/run_gemini.py b/benchmarks/olmocr/bench/runners/run_gemini.py new file mode 100644 index 0000000..d038bea --- /dev/null +++ b/benchmarks/olmocr/bench/runners/run_gemini.py @@ -0,0 +1,102 @@ +import base64 +import os + +from google import genai +from google.genai import types + +from benchmarks.olmocr.data.renderpdf import render_pdf_to_base64png + +_client = None + +DEFAULT_MODEL = "gemini-3.7-flash" + + +def _get_client(): + global _client + if _client is None: + api_key = ( + os.getenv("GEMINI_API_KEY") + or os.getenv("GEMINI_KEY") + or os.getenv("GOOGLE_API_KEY") + ) + if not api_key: + raise SystemExit( + "GEMINI_API_KEY not set — get it from https://aistudio.google.com/apikey" + ) + # Cached deliberately: building a client per call gets it collected + # mid-flight ("Cannot send a request, as the client has been closed"). + _client = genai.Client(api_key=api_key) + return _client + + +def _thinking_floor(model: str) -> str: + """Lowest thinking_level the model accepts. + + gemini-3.7+ flash rejects MINIMAL server-side ("Thinking level MINIMAL is + not supported for this model") and the SDK enum has no off/none, so "low" is + its floor. Older 3.x flash still takes "minimal"; Pro models reject + "minimal" too. Override with BENCH_GEMINI_FLOOR. + """ + override = os.getenv("BENCH_GEMINI_FLOOR") + if override: + return override + m = model.lower() + if "pro" in m or any(v in m for v in ("3.7", "3.8", "3.9", "4.")): + return "low" + return "minimal" + + +def run_gemini( + pdf_path: str, + page_num: int = 1, + model: str = DEFAULT_MODEL, + target_longest_image_dim: int = 2048, +) -> str: + """Convert a PDF page to markdown via any vision-capable Gemini model. + + Generalises run_gemini_pro_31, which pinned both the model and its thinking + config. Same prompt, so scores stay comparable across models. + """ + image_base64 = render_pdf_to_base64png( + pdf_path, page_num=page_num, target_longest_image_dim=target_longest_image_dim + ) + image_bytes = base64.b64decode(image_base64) + + client = _get_client() + + prompt = ( + "Below is the image of one page of a PDF document. " + "Just return the plain text representation of this document as if you were reading it naturally.\n" + "Turn equations into LaTeX using \\( \\) for inline math and \\[ \\] for display math. " + "Never describe equations in words — always use LaTeX notation. " + "Turn tables into markdown format.\n" + "Remove the headers and footers completely — do not include any text " + "that appears at the very top or very bottom of the page outside the main body content. " + "This includes page numbers, journal names, author names in running headers, " + "copyright lines, DOI lines, citation requests, institutional addresses in margins, " + "and download dates. Keep references and footnotes that are part of the body.\n" + "For multi-column layouts, read each column top to bottom before moving to the next.\n" + "Read any natural handwriting.\n" + "This is likely one page out of several in the document, so be sure to preserve " + "any sentences that come from the previous page, or continue onto the next page, exactly as they are.\n" + "If there is no text at all that you think you should read, you can output null.\n" + "Do not hallucinate." + ) + + image_part = types.Part.from_bytes(data=image_bytes, mime_type="image/png") + + config = types.GenerateContentConfig( + temperature=0.0, + thinking_config=types.ThinkingConfig(thinking_level=_thinking_floor(model)), + ) + + response = client.models.generate_content( + model=model, + contents=[prompt, image_part], + config=config, + ) + + raw = response.text if hasattr(response, "text") else None + if raw is None or raw.strip().lower() in ("null", "none", "n/a", ""): + return "" + return raw diff --git a/benchmarks/olmocr/olmocr_bench_fireworks.py b/benchmarks/olmocr/olmocr_bench_fireworks.py new file mode 100644 index 0000000..0586801 --- /dev/null +++ b/benchmarks/olmocr/olmocr_bench_fireworks.py @@ -0,0 +1,279 @@ +""" +OlmOCR Benchmark for Fireworks-hosted models (default: Inkling). + +Mirrors olmocr_bench.py (interfaze) but routes pages through Fireworks' +OpenAI-compatible endpoint. + +Usage: + uv run -m benchmarks.olmocr.olmocr_bench_fireworks + uv run -m benchmarks.olmocr.olmocr_bench_fireworks --sample + uv run -m benchmarks.olmocr.olmocr_bench_fireworks --skip-generation + uv run -m benchmarks.olmocr.olmocr_bench_fireworks --generate-only + uv run -m benchmarks.olmocr.olmocr_bench_fireworks --model accounts/fireworks/models/inkling-small +""" + +import argparse +import asyncio +import json +import os +import re +import sys +from pathlib import Path + +from dotenv import load_dotenv +from huggingface_hub import hf_hub_download +from tqdm.asyncio import tqdm_asyncio + +load_dotenv() + +PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent +SAMPLE_DATA_DIR = Path(__file__).resolve().parent / "bench" / "sample_data" +FULL_DATA_DIR = Path(__file__).resolve().parent / "bench" / "full_data" + +INKLING = "accounts/fireworks/models/inkling" +MODEL = INKLING +# Candidate name = the output subdirectory the olmOCR scorer reads. Derived from +# the model so inkling and inkling-small don't overwrite each other. +CANDIDATE_NAME = "inkling" +RATE_LIMIT = 25 +MAX_RETRIES = 3 + +# Cap on requests in flight. RATE_LIMIT bounds how many requests *start* per +# second but not how many are outstanding — all ~1400 pages launched at once +# queue server-side and eventually 429 after minutes of latency. Override with +# BENCH_MAX_IN_FLIGHT. +MAX_IN_FLIGHT = int(os.getenv("BENCH_MAX_IN_FLIGHT", "8")) + +HF_REPO = "allenai/olmOCR-bench" +SPLITS = [ + "arxiv_math", + "headers_footers", + "long_tiny_text", + "multi_column", + "old_scans", + "old_scans_math", + "table_tests", +] + +sys.path.insert(0, str(PROJECT_ROOT)) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + + +def model_slug(model: str) -> str: + """accounts/fireworks/models/inkling-small -> inkling-small""" + return re.sub(r"[^a-z0-9]+", "-", model.rsplit("/", 1)[-1].lower()).strip("-") + + +class RateLimiter: + def __init__(self, rate: int): + self.rate = rate + self.tokens = rate + self.last_refill = 0.0 + self._lock = asyncio.Lock() + + async def acquire(self): + while True: + async with self._lock: + now = asyncio.get_running_loop().time() + elapsed = now - self.last_refill + self.tokens = min(self.rate, self.tokens + elapsed * self.rate) + self.last_refill = now + if self.tokens >= 1: + self.tokens -= 1 + return + await asyncio.sleep(1 / self.rate) + + +def download_full_dataset(): + data_dir = FULL_DATA_DIR + pdf_dir = data_dir / "pdfs" + all_pdfs = set() + for split in SPLITS: + jsonl_dest = data_dir / f"{split}.jsonl" + if jsonl_dest.exists(): + with open(jsonl_dest) as f: + tests = [json.loads(l) for l in f if l.strip()] + else: + print(f" Downloading {split}.jsonl...") + src = hf_hub_download( + HF_REPO, f"bench_data/{split}.jsonl", repo_type="dataset" + ) + with open(src) as f: + tests = [json.loads(l) for l in f if l.strip()] + data_dir.mkdir(parents=True, exist_ok=True) + with open(jsonl_dest, "w") as f: + for t in tests: + f.write(json.dumps(t) + "\n") + print(f" {split}: {len(tests)} tests") + for t in tests: + all_pdfs.add(t["pdf"]) + + print(f"\n Total unique PDFs to download: {len(all_pdfs)}") + downloaded = 0 + skipped = 0 + for pdf_rel in sorted(all_pdfs): + local_path = pdf_dir / pdf_rel + if local_path.exists(): + skipped += 1 + continue + local_path.parent.mkdir(parents=True, exist_ok=True) + try: + src = hf_hub_download( + HF_REPO, f"bench_data/pdfs/{pdf_rel}", repo_type="dataset" + ) + os.symlink(src, str(local_path)) + downloaded += 1 + except Exception as e: + print(f" Failed to download {pdf_rel}: {e}") + print(f" PDFs: {downloaded} downloaded, {skipped} already existed") + return data_dir + + +async def process_page(pdf_path, page_num, output_path, rate_limiter, sem): + async with sem: + return await _process_page(pdf_path, page_num, output_path, rate_limiter) + + +async def _process_page(pdf_path, page_num, output_path, rate_limiter): + from olmocr.bench.runners.run_fireworks import run_fireworks + + for attempt in range(MAX_RETRIES): + await rate_limiter.acquire() + try: + result = await asyncio.to_thread(run_fireworks, pdf_path, page_num, MODEL) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + f.write(result) + return True + except Exception as e: + if attempt < MAX_RETRIES - 1: + await asyncio.sleep(2**attempt) + else: + print( + f"Failed after {MAX_RETRIES} attempts: {pdf_path} page {page_num}: {e}" + ) + return False + + +async def generate_outputs(data_dir: Path): + pdf_folder = data_dir / "pdfs" + output_folder = data_dir / CANDIDATE_NAME + + pdf_pages = set() + for jsonl_file in data_dir.glob("*.jsonl"): + with open(jsonl_file) as f: + for line in f: + line = line.strip() + if not line: + continue + t = json.loads(line) + pdf_pages.add((t["pdf"], t["page"])) + + print(f"Found {len(pdf_pages)} unique (pdf, page) pairs to process") + + rate_limiter = RateLimiter(RATE_LIMIT) + sem = asyncio.Semaphore(MAX_IN_FLIGHT) + tasks = [] + for pdf_rel, page in sorted(pdf_pages): + pdf_path = str(pdf_folder / pdf_rel) + if not os.path.exists(pdf_path): + continue + base_name = os.path.splitext(os.path.basename(pdf_rel))[0] + parent_dir = os.path.dirname(pdf_rel) + md_filename = f"{base_name}_pg{page}_repeat1.md" + if parent_dir: + out_path = str(output_folder / parent_dir / md_filename) + else: + out_path = str(output_folder / md_filename) + if os.path.exists(out_path): + continue + tasks.append(process_page(pdf_path, page, out_path, rate_limiter, sem)) + + if not tasks: + print("All outputs already exist, skipping generation.") + return True + print(f"Processing {len(tasks)} pages...") + results = await tqdm_asyncio.gather( + *tasks, desc=f"Generating {CANDIDATE_NAME} outputs" + ) + num_success = sum(1 for r in results if r) + num_failed = len(results) - num_success + print(f"Done: {num_success} succeeded, {num_failed} failed") + return num_failed == 0 + + +class _Tee: + """olmOCR-bench prints its scores and saves nothing, so mirror stdout to a + log file — otherwise the numbers exist only in whatever terminal ran it and + scripts/report_scores.py has nothing to read.""" + + def __init__(self, *streams): + self.streams = streams + + def write(self, data): + for s in self.streams: + s.write(data) + s.flush() + + def flush(self): + for s in self.streams: + s.flush() + + +def run_evaluation(data_dir: Path): + from olmocr.bench.benchmark import main as bench_main + + sys.argv = [ + "benchmark", + "--dir", + str(data_dir), + "--candidate", + CANDIDATE_NAME, + "--force", + ] + log_path = PROJECT_ROOT / "logs" / f"olmocr_{CANDIDATE_NAME}.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + real_stdout = sys.stdout + with open(log_path, "w", encoding="utf-8") as fh: + sys.stdout = _Tee(real_stdout, fh) + try: + bench_main() + finally: + sys.stdout = real_stdout + print(f"\nolmOCR scores also written to {log_path}") + + +async def main(): + global MODEL, CANDIDATE_NAME + parser = argparse.ArgumentParser( + description="Run OlmOCR benchmark against a Fireworks-hosted model" + ) + parser.add_argument( + "--model", default=INKLING, help="Fireworks model id (accounts/fireworks/...)" + ) + parser.add_argument("--sample", action="store_true") + parser.add_argument("--skip-generation", action="store_true") + parser.add_argument("--generate-only", action="store_true") + args = parser.parse_args() + + MODEL = args.model + CANDIDATE_NAME = model_slug(MODEL) + + if args.sample: + data_dir = SAMPLE_DATA_DIR + print("=== Using sample data ===") + else: + print("=== Downloading full olmOCR-bench dataset from HuggingFace ===") + data_dir = download_full_dataset() + + if not args.skip_generation: + print(f"\n=== Generating {CANDIDATE_NAME} outputs ===") + await generate_outputs(data_dir) + + if not args.generate_only: + print("\n=== Running OlmOCR Benchmark Evaluation ===") + run_evaluation(data_dir) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/benchmarks/olmocr/olmocr_bench_gemini.py b/benchmarks/olmocr/olmocr_bench_gemini.py new file mode 100644 index 0000000..415e77b --- /dev/null +++ b/benchmarks/olmocr/olmocr_bench_gemini.py @@ -0,0 +1,280 @@ +""" +OlmOCR Benchmark for Gemini models (native google-genai, default 3.7 Flash). + +Mirrors olmocr_bench.py but routes pages through the native Gemini API, so +the methodology matches the repo's other --provider gemini runs (thinking_level +rather than OpenRouter's reasoning.effort). + +Usage: + uv run -m benchmarks.olmocr.olmocr_bench_gemini + uv run -m benchmarks.olmocr.olmocr_bench_gemini --sample + uv run -m benchmarks.olmocr.olmocr_bench_gemini --skip-generation + uv run -m benchmarks.olmocr.olmocr_bench_gemini --generate-only + uv run -m benchmarks.olmocr.olmocr_bench_gemini --model accounts/fireworks/models/inkling-small +""" + +import argparse +import asyncio +import json +import os +import re +import sys +from pathlib import Path + +from dotenv import load_dotenv +from huggingface_hub import hf_hub_download +from tqdm.asyncio import tqdm_asyncio + +load_dotenv() + +PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent +SAMPLE_DATA_DIR = Path(__file__).resolve().parent / "bench" / "sample_data" +FULL_DATA_DIR = Path(__file__).resolve().parent / "bench" / "full_data" + +DEFAULT_MODEL = "gemini-3.7-flash" +MODEL = DEFAULT_MODEL +# Candidate name = the output subdirectory the olmOCR scorer reads. Derived from +# the model so inkling and inkling-small don't overwrite each other. +CANDIDATE_NAME = "gemini-3-7-flash" +RATE_LIMIT = 25 +MAX_RETRIES = 3 + +# Cap on requests in flight. RATE_LIMIT bounds how many requests *start* per +# second but not how many are outstanding — all ~1400 pages launched at once +# queue server-side and eventually get rate-limited. Override with +# BENCH_MAX_IN_FLIGHT. +MAX_IN_FLIGHT = int(os.getenv("BENCH_MAX_IN_FLIGHT", "8")) + +HF_REPO = "allenai/olmOCR-bench" +SPLITS = [ + "arxiv_math", + "headers_footers", + "long_tiny_text", + "multi_column", + "old_scans", + "old_scans_math", + "table_tests", +] + +sys.path.insert(0, str(PROJECT_ROOT)) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + + +def model_slug(model: str) -> str: + """accounts/fireworks/models/inkling-small -> inkling-small""" + return re.sub(r"[^a-z0-9]+", "-", model.rsplit("/", 1)[-1].lower()).strip("-") + + +class RateLimiter: + def __init__(self, rate: int): + self.rate = rate + self.tokens = rate + self.last_refill = 0.0 + self._lock = asyncio.Lock() + + async def acquire(self): + while True: + async with self._lock: + now = asyncio.get_running_loop().time() + elapsed = now - self.last_refill + self.tokens = min(self.rate, self.tokens + elapsed * self.rate) + self.last_refill = now + if self.tokens >= 1: + self.tokens -= 1 + return + await asyncio.sleep(1 / self.rate) + + +def download_full_dataset(): + data_dir = FULL_DATA_DIR + pdf_dir = data_dir / "pdfs" + all_pdfs = set() + for split in SPLITS: + jsonl_dest = data_dir / f"{split}.jsonl" + if jsonl_dest.exists(): + with open(jsonl_dest) as f: + tests = [json.loads(l) for l in f if l.strip()] + else: + print(f" Downloading {split}.jsonl...") + src = hf_hub_download( + HF_REPO, f"bench_data/{split}.jsonl", repo_type="dataset" + ) + with open(src) as f: + tests = [json.loads(l) for l in f if l.strip()] + data_dir.mkdir(parents=True, exist_ok=True) + with open(jsonl_dest, "w") as f: + for t in tests: + f.write(json.dumps(t) + "\n") + print(f" {split}: {len(tests)} tests") + for t in tests: + all_pdfs.add(t["pdf"]) + + print(f"\n Total unique PDFs to download: {len(all_pdfs)}") + downloaded = 0 + skipped = 0 + for pdf_rel in sorted(all_pdfs): + local_path = pdf_dir / pdf_rel + if local_path.exists(): + skipped += 1 + continue + local_path.parent.mkdir(parents=True, exist_ok=True) + try: + src = hf_hub_download( + HF_REPO, f"bench_data/pdfs/{pdf_rel}", repo_type="dataset" + ) + os.symlink(src, str(local_path)) + downloaded += 1 + except Exception as e: + print(f" Failed to download {pdf_rel}: {e}") + print(f" PDFs: {downloaded} downloaded, {skipped} already existed") + return data_dir + + +async def process_page(pdf_path, page_num, output_path, rate_limiter, sem): + async with sem: + return await _process_page(pdf_path, page_num, output_path, rate_limiter) + + +async def _process_page(pdf_path, page_num, output_path, rate_limiter): + from olmocr.bench.runners.run_gemini import run_gemini + + for attempt in range(MAX_RETRIES): + await rate_limiter.acquire() + try: + result = await asyncio.to_thread(run_gemini, pdf_path, page_num, MODEL) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + f.write(result) + return True + except Exception as e: + if attempt < MAX_RETRIES - 1: + await asyncio.sleep(2**attempt) + else: + print( + f"Failed after {MAX_RETRIES} attempts: {pdf_path} page {page_num}: {e}" + ) + return False + + +async def generate_outputs(data_dir: Path): + pdf_folder = data_dir / "pdfs" + output_folder = data_dir / CANDIDATE_NAME + + pdf_pages = set() + for jsonl_file in data_dir.glob("*.jsonl"): + with open(jsonl_file) as f: + for line in f: + line = line.strip() + if not line: + continue + t = json.loads(line) + pdf_pages.add((t["pdf"], t["page"])) + + print(f"Found {len(pdf_pages)} unique (pdf, page) pairs to process") + + rate_limiter = RateLimiter(RATE_LIMIT) + sem = asyncio.Semaphore(MAX_IN_FLIGHT) + tasks = [] + for pdf_rel, page in sorted(pdf_pages): + pdf_path = str(pdf_folder / pdf_rel) + if not os.path.exists(pdf_path): + continue + base_name = os.path.splitext(os.path.basename(pdf_rel))[0] + parent_dir = os.path.dirname(pdf_rel) + md_filename = f"{base_name}_pg{page}_repeat1.md" + if parent_dir: + out_path = str(output_folder / parent_dir / md_filename) + else: + out_path = str(output_folder / md_filename) + if os.path.exists(out_path): + continue + tasks.append(process_page(pdf_path, page, out_path, rate_limiter, sem)) + + if not tasks: + print("All outputs already exist, skipping generation.") + return True + print(f"Processing {len(tasks)} pages...") + results = await tqdm_asyncio.gather( + *tasks, desc=f"Generating {CANDIDATE_NAME} outputs" + ) + num_success = sum(1 for r in results if r) + num_failed = len(results) - num_success + print(f"Done: {num_success} succeeded, {num_failed} failed") + return num_failed == 0 + + +class _Tee: + """olmOCR-bench prints its scores and saves nothing, so mirror stdout to a + log file — otherwise the numbers exist only in whatever terminal ran it and + scripts/report_scores.py has nothing to read.""" + + def __init__(self, *streams): + self.streams = streams + + def write(self, data): + for s in self.streams: + s.write(data) + s.flush() + + def flush(self): + for s in self.streams: + s.flush() + + +def run_evaluation(data_dir: Path): + from olmocr.bench.benchmark import main as bench_main + + sys.argv = [ + "benchmark", + "--dir", + str(data_dir), + "--candidate", + CANDIDATE_NAME, + "--force", + ] + log_path = PROJECT_ROOT / "logs" / f"olmocr_{CANDIDATE_NAME}.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + real_stdout = sys.stdout + with open(log_path, "w", encoding="utf-8") as fh: + sys.stdout = _Tee(real_stdout, fh) + try: + bench_main() + finally: + sys.stdout = real_stdout + print(f"\nolmOCR scores also written to {log_path}") + + +async def main(): + global MODEL, CANDIDATE_NAME + parser = argparse.ArgumentParser( + description="Run OlmOCR benchmark against a Gemini model (native API)" + ) + parser.add_argument( + "--model", default=DEFAULT_MODEL, help="Gemini model id (e.g. gemini-3.7-flash)" + ) + parser.add_argument("--sample", action="store_true") + parser.add_argument("--skip-generation", action="store_true") + parser.add_argument("--generate-only", action="store_true") + args = parser.parse_args() + + MODEL = args.model + CANDIDATE_NAME = model_slug(MODEL) + + if args.sample: + data_dir = SAMPLE_DATA_DIR + print("=== Using sample data ===") + else: + print("=== Downloading full olmOCR-bench dataset from HuggingFace ===") + data_dir = download_full_dataset() + + if not args.skip_generation: + print(f"\n=== Generating {CANDIDATE_NAME} outputs ===") + await generate_outputs(data_dir) + + if not args.generate_only: + print("\n=== Running OlmOCR Benchmark Evaluation ===") + run_evaluation(data_dir) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/benchmarks/spider2_lite/spider2_lite.py b/benchmarks/spider2_lite/spider2_lite.py index 5dd8840..74416c7 100644 --- a/benchmarks/spider2_lite/spider2_lite.py +++ b/benchmarks/spider2_lite/spider2_lite.py @@ -79,6 +79,22 @@ # Config ---------------------------------------------------------------------- +# Provider/model, set from the CLI. interfaze is the default so existing +# invocations and output paths are unchanged. +PROVIDER = "interfaze" +MODEL = "interfaze-beta" + +# Per-query wall-clock cap during scoring — see execute_sqlite. +QUERY_TIMEOUT_S = 120.0 + +# reasoning.effort sent to OpenRouter-hosted models. Override per run. +OPENROUTER_EFFORT = os.getenv("BENCH_OPENROUTER_EFFORT", "none") + +_GEMINI_CLIENT = None +# Gemini thinking_level: "auto" = the model's floor (3.7 rejects minimal, so +# "low"), or an explicit minimal|low|medium|high. Set from the CLI. +GEMINI_LEVEL = "auto" + REASONING_EFFORT = None TEMPERATURE = 0.0 RATE_LIMIT = 8 @@ -108,6 +124,7 @@ # Utilities shared with other benches # ----------------------------------------------------------------------------- + class RateLimiter: def __init__(self, rate: int): self.rate = rate @@ -147,6 +164,7 @@ async def append(self, record: dict): # Dataset loading # ----------------------------------------------------------------------------- + def load_local_examples() -> list[dict]: if not ALL_EXAMPLES.exists(): raise FileNotFoundError( @@ -205,6 +223,12 @@ def build_prompt(example: dict) -> str: _SQL_FENCE = re.compile(r"```sql\s*\n(.*?)```", re.IGNORECASE | re.DOTALL) _ANY_FENCE = re.compile(r"```\s*\n?(.*?)```", re.DOTALL) +# Unclosed fence: an opener with no terminator. Some models (deepseek-v4-flash +# on 57/135 Spider2 examples) emit complete SQL ending in ';' but never close +# the block. Without this, the whole response — opening fence included — was +# handed to sqlite as SQL and failed with `unrecognized token: "```sql`, +# scoring a valid query as wrong. +_OPEN_FENCE = re.compile(r"```(?:sql)?[ \t]*\r?\n(.*)\Z", re.IGNORECASE | re.DOTALL) def extract_sql(text: str) -> str: @@ -217,6 +241,9 @@ def extract_sql(text: str) -> str: if m: return m.group(1).strip() m = _ANY_FENCE.search(text) + if m: + return m.group(1).strip() + m = _OPEN_FENCE.search(text) if m: return m.group(1).strip() return text.strip() @@ -226,6 +253,7 @@ def extract_sql(text: str) -> str: # Checkpoint helpers # ----------------------------------------------------------------------------- + def load_completed_ids(path: Path) -> set[str]: if not path.exists(): return set() @@ -265,8 +293,82 @@ def load_records(path: Path) -> list[dict]: # Prediction # ----------------------------------------------------------------------------- -async def process_example(example: dict, rate_limiter: RateLimiter, - writer: JsonlWriter, progress: dict) -> dict | None: + +def invoke_model(messages: list[dict]): + """Dispatch to the configured provider — both endpoints are OpenAI-shaped. + + Inkling always thinks: reasoning_effort="none" is its floor, so a Fireworks + run is not directly comparable to the interfaze reasoning-off run.""" + if PROVIDER == "gemini": + # google-genai returns resp.text, but every caller here expects an + # OpenAI-shaped object, so wrap it rather than special-casing downstream. + from google import genai + from google.genai import types + + key = ( + os.getenv("GEMINI_API_KEY") + or os.getenv("GEMINI_KEY") + or os.getenv("GOOGLE_API_KEY") + ) + if not key: + raise ValueError("GEMINI_API_KEY missing from .env") + if GEMINI_LEVEL != "auto": + level = GEMINI_LEVEL + elif any(v in MODEL.lower() for v in ("3.7", "3.8", "3.9", "4.", "pro")): + level = "low" + else: + level = "minimal" + # Cache the client: constructing one per call gets it garbage-collected + # mid-flight ("Cannot send a request, as the client has been closed"). + global _GEMINI_CLIENT + if _GEMINI_CLIENT is None: + _GEMINI_CLIENT = genai.Client(api_key=key) + resp = _GEMINI_CLIENT.models.generate_content( + model=MODEL, + contents=[messages[0]["content"]], + config=types.GenerateContentConfig( + temperature=TEMPERATURE, + thinking_config=types.ThinkingConfig(thinking_level=level), + ), + ) + + class _Shim: + def __init__(self, text, rid): + self.choices = [ + type("C", (), {"message": type("M", (), {"content": text})()})() + ] + self.id = rid + + return _Shim(resp.text or "", getattr(resp, "response_id", "") or "") + if PROVIDER == "openrouter": + from src.commons_openrouter import invoke_openrouter + + return invoke_openrouter( + messages, + model=MODEL, + temperature=TEMPERATURE, + extra_body={"reasoning": {"effort": OPENROUTER_EFFORT}}, + ) + if PROVIDER == "fireworks": + from src.commons_fireworks import invoke_fireworks + + return invoke_fireworks( + messages, + model=MODEL, + reasoning_effort="none", + temperature=TEMPERATURE, + ) + return invoke_interfaze( + messages, + model=MODEL, + reasoning_effort=REASONING_EFFORT, + temperature=TEMPERATURE, + ) + + +async def process_example( + example: dict, rate_limiter: RateLimiter, writer: JsonlWriter, progress: dict +) -> dict | None: instance_id = example["instance_id"] prompt = build_prompt(example) messages = [{"role": "user", "content": prompt}] @@ -276,12 +378,7 @@ async def process_example(example: dict, rate_limiter: RateLimiter, await rate_limiter.acquire() start = time.perf_counter() try: - response = await asyncio.to_thread( - invoke_interfaze, - messages, - reasoning_effort=REASONING_EFFORT, - temperature=TEMPERATURE, - ) + response = await asyncio.to_thread(invoke_model, messages) latency_ms = int((time.perf_counter() - start) * 1000) content = (response.choices[0].message.content or "").strip() request_id = getattr(response, "id", None) @@ -337,8 +434,10 @@ async def run_prediction(examples: list[dict], limit: int | None): if limit is not None: pending = pending[:limit] print(f"--limit applied: will run at most {limit} example(s)") - print(f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {RESPONSES_PATH})") + print( + f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " + f"(checkpoint: {RESPONSES_PATH})" + ) if not pending: return @@ -350,8 +449,10 @@ async def run_prediction(examples: list[dict], limit: int | None): await tqdm_asyncio.gather(*tasks, desc="spider2-lite/local") except Exception: traceback.print_exc() - print(f"\nPrediction finished: {progress['done']}/{progress['total']} answered, " - f"{progress['failed']} failed.") + print( + f"\nPrediction finished: {progress['done']}/{progress['total']} answered, " + f"{progress['failed']} failed." + ) # ----------------------------------------------------------------------------- @@ -362,6 +463,7 @@ async def run_prediction(examples: list[dict], limit: int | None): # condition_cols / ignore_order flags, multi-gold). # ----------------------------------------------------------------------------- + def _normalize(v): return 0 if pd.isna(v) else v @@ -389,8 +491,9 @@ def _vectors_match(v1, v2, ignore_order: bool, tol: float = 1e-2) -> bool: return True -def compare_table(pred: pd.DataFrame, gold: pd.DataFrame, - condition_cols, ignore_order: bool) -> int: +def compare_table( + pred: pd.DataFrame, gold: pd.DataFrame, condition_cols, ignore_order: bool +) -> int: if condition_cols: if not isinstance(condition_cols, (list, tuple)): condition_cols = [condition_cols] @@ -405,8 +508,12 @@ def compare_table(pred: pd.DataFrame, gold: pd.DataFrame, return 1 -def compare_multi(pred: pd.DataFrame, golds: list[pd.DataFrame], - multi_condition_cols, ignore_order: bool) -> int: +def compare_multi( + pred: pd.DataFrame, + golds: list[pd.DataFrame], + multi_condition_cols, + ignore_order: bool, +) -> int: if not golds: return 0 if multi_condition_cols in (None, [], [[]], [None]): @@ -437,13 +544,26 @@ def execute_sqlite(db_path: Path, sql: str) -> tuple[bool, pd.DataFrame | str]: We copy the on-disk DB into :memory: (same pattern as the official evaluate.py) — faster for repeated queries and isolates writes. + + Queries are capped at QUERY_TIMEOUT_S. A model can emit SQL that never + finishes (an unbounded cross join is the usual culprit), and without a cap + that single example hangs the whole scoring pass at 100% CPU. A query that + can't complete in the budget is scored as a failed execution, which is what + the official evaluate.py's func_timeout does too. """ try: disk = sqlite3.connect(str(db_path)) mem = sqlite3.connect(":memory:") try: disk.backup(mem) + deadline = time.monotonic() + QUERY_TIMEOUT_S + # Called every N VM instructions; non-zero aborts the query, which + # surfaces as sqlite3.OperationalError('interrupted'). + mem.set_progress_handler( + lambda: 1 if time.monotonic() > deadline else 0, 10_000 + ) df = pd.read_sql_query(sql, mem) + mem.set_progress_handler(None, 0) return True, df finally: mem.close() @@ -473,7 +593,8 @@ def evaluate_record(record: dict, eval_std: dict) -> dict: db_path = SQLITE_DB_DIR / f"{record['db']}.sqlite" if not db_path.exists(): return { - "instance_id": instance_id, "score": 0, + "instance_id": instance_id, + "score": 0, "error": f"missing sqlite db: {db_path}", } pred_sql = record.get("pred_sql") or "" @@ -513,6 +634,20 @@ def run_evaluation(total_local: int) -> None: sys.exit(1) eval_std = load_eval_standard() + # Re-derive SQL from the stored response so --evaluate-only picks up + # extractor fixes without paying for new predictions (same convention as + # the mmmlu/gpqa runners re-running their answer parsers). + n_rewritten = 0 + for rec in records: + if not rec.get("response"): + continue + fresh = extract_sql(rec["response"]) + if fresh and fresh != rec.get("pred_sql"): + rec["pred_sql"] = fresh + n_rewritten += 1 + if n_rewritten: + print(f"Re-extracted SQL for {n_rewritten} record(s) with the current parser.") + results: list[dict] = [] for rec in tqdm(records, desc="Evaluating"): res = evaluate_record(rec, eval_std) @@ -527,7 +662,9 @@ def run_evaluation(total_local: int) -> None: accuracy_of_subset = correct / total_local if total_local else 0.0 print(f"\n{'=' * 60}") - print(f"Spider 2.0-Lite — SQLite subset (Interfaze, reasoning={REASONING_EFFORT})") + print( + f"Spider 2.0-Lite — SQLite subset ({PROVIDER}/{MODEL}, reasoning={REASONING_EFFORT})" + ) print(f"{'=' * 60}") print(f"Correct : {correct}/{total}") print(f"Accuracy (of predicted): {accuracy_of_evaluated:.4f}") @@ -552,7 +689,9 @@ def run_evaluation(total_local: int) -> None: "total_local_subset": total_local, "subset": "local (SQLite)", "benchmark": "Spider 2.0-Lite", - "reasoning_effort": REASONING_EFFORT, + "provider": PROVIDER, + "model": MODEL, + "reasoning_effort": "none" if PROVIDER == "fireworks" else REASONING_EFFORT, "temperature": TEMPERATURE, "per_example": results, } @@ -566,14 +705,63 @@ def run_evaluation(total_local: int) -> None: # Entrypoint # ----------------------------------------------------------------------------- + +def set_paths() -> None: + """Per provider+model output paths so runs don't share a checkpoint. The + interfaze default keeps the original `spider2_lite_local` tag.""" + global TAG, RESPONSES_PATH, PRED_SQL_DIR, METRICS_PATH + if PROVIDER != "interfaze": + slug = re.sub(r"[^a-z0-9]+", "-", MODEL.rsplit("/", 1)[-1].lower()).strip("-") + TAG = f"spider2_lite_local_{PROVIDER}_{slug}" + if GEMINI_LEVEL != "auto": + TAG += f"_thinking{GEMINI_LEVEL}" + RESPONSES_PATH = RESULTS_DIR / f"{TAG}_responses.jsonl" + PRED_SQL_DIR = RESULTS_DIR / f"{TAG}_sql" + METRICS_PATH = RESULTS_DIR / f"{TAG}_metrics.json" + + def main() -> None: - parser = argparse.ArgumentParser(description="Spider 2.0-Lite SQLite subset (Interfaze)") + global PROVIDER, MODEL, GEMINI_LEVEL + parser = argparse.ArgumentParser( + description="Spider 2.0-Lite SQLite subset (Interfaze or Fireworks)" + ) + parser.add_argument( + "--provider", default="interfaze", choices=["interfaze", "fireworks", "openrouter", "gemini"] + ) + parser.add_argument( + "--thinking-level", + default="auto", + choices=["auto", "minimal", "low", "medium", "high"], + help="Gemini thinking_level; auto = the model's floor. Non-auto values " + "get their own output tag so a floor run is never overwritten.", + ) + parser.add_argument( + "--model", + default=None, + help="Model id. Defaults to interfaze-beta, or Inkling for --provider fireworks.", + ) parser.add_argument("--predict-only", action="store_true") parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument("--limit", type=int, default=None, - help="Only predict the first N pending examples") + parser.add_argument( + "--limit", + type=int, + default=None, + help="Only predict the first N pending examples", + ) args = parser.parse_args() + PROVIDER = args.provider + GEMINI_LEVEL = args.thinking_level + if args.model: + MODEL = args.model + elif PROVIDER == "fireworks": + MODEL = "accounts/fireworks/models/inkling" + elif PROVIDER == "openrouter": + MODEL = "thinkingmachines/inkling-small" + elif PROVIDER == "gemini": + MODEL = "gemini-3.7-flash" + set_paths() + examples = load_local_examples() print(f"Loaded {len(examples)} local (SQLite) examples from {ALL_EXAMPLES.name}") diff --git a/pyproject.toml b/pyproject.toml index e3f5b95..7a0bf4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,4 +41,21 @@ dependencies = [ "langchain>=1.2.11", "olmocr==0.4.27", "pymupdf>=1.27.2", + "jiwer>=3.0.0", + # olmOCR-bench scoring: fuzzy text matching + headless KaTeX equation render + # (needs `uv run python -m playwright install chromium` once). + "fuzzysearch>=0.7.3", + "playwright>=1.44.0", + "gdown>=6.1.0", + "google-genai>=2.18.1", ] + +[dependency-groups] +dev = [ + "pytest>=8.0.0", + "pytest-asyncio>=0.24.0", +] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] diff --git a/scripts/bfcl.sh b/scripts/bfcl.sh new file mode 100755 index 0000000..5a74cc2 --- /dev/null +++ b/scripts/bfcl.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Thin wrapper around BFCL's CLI that fixes up the environment first. +# +# scripts/bfcl.sh generate --model fireworks-qwen3p7-plus-FC --test-category simple_python +# scripts/bfcl.sh evaluate --model fireworks-qwen3p7-plus-FC --test-category simple_python +# +# Two reasons this exists rather than calling `bfcl` directly: +# +# 1. BFCL derives DOTENV_PATH from BFCL_PROJECT_ROOT, which it reads from the +# *process* environment — so BFCL_PROJECT_ROOT cannot be supplied by the +# .env it is used to locate. It has to be exported before the CLI starts. +# 2. This project's .env spells the Fireworks key lowercase, while BFCL's +# handler does a bare os.getenv("FIREWORKS_API_KEY"). We normalise here. +# +# Run scripts/setup_bfcl.sh once first. + +set -euo pipefail +cd "$(dirname "$0")/.." +ROOT="$PWD" + +VENV="$ROOT/external/bfcl-venv" +RUNS="$ROOT/external/bfcl-runs" + +if [ ! -x "$VENV/bin/bfcl" ]; then + echo "bfcl not installed — run scripts/setup_bfcl.sh first" >&2 + exit 1 +fi + +export BFCL_PROJECT_ROOT="$RUNS" +export FIREWORKS_API_KEY="${FIREWORKS_API_KEY:-$( + grep -iE '^[[:space:]]*fireworks_api_key[[:space:]]*=' "$ROOT/.env" 2>/dev/null \ + | head -1 | cut -d= -f2- | tr -d ' "'"'" || true +)}" +export OPENROUTER_API_KEY="${OPENROUTER_API_KEY:-$( + grep -iE '^[[:space:]]*openrouter_api_key[[:space:]]*=' "$ROOT/.env" 2>/dev/null \ + | head -1 | cut -d= -f2- | tr -d ' "'"'" || true +)}" +# See setup_bfcl.sh — needed only so the parent handler's client can be built. +export OPENAI_API_KEY="${OPENAI_API_KEY:-placeholder-unused-by-fireworks-handler}" + +if [ -z "$FIREWORKS_API_KEY" ] && [ -z "$OPENROUTER_API_KEY" ]; then + echo "Neither FIREWORKS_API_KEY nor OPENROUTER_API_KEY found in $ROOT/.env" >&2 + exit 1 +fi + +cd "$RUNS" +exec "$VENV/bin/bfcl" "$@" diff --git a/scripts/plot_gemini_vs_deepseek.py b/scripts/plot_gemini_vs_deepseek.py new file mode 100644 index 0000000..6a95c20 --- /dev/null +++ b/scripts/plot_gemini_vs_deepseek.py @@ -0,0 +1,277 @@ +""" +Gemini-3.5-flash vs DeepSeek-V4-Flash across the benchmark suite. + +Bars, not a radar: DeepSeek V4 Flash is text-only, so 6 of 9 benchmarks are not +runnable for it at all. On a radar those axes would collapse to the centre and +read as "scored badly"; as bars, an absent bar can be labelled for what it is +("no image input") and never be mistaken for a zero. + +Gemini figures come from the CSVs in the repo root (the team's own runs). +DeepSeek figures are measured by this repo. + + uv run python scripts/plot_gemini_vs_deepseek.py +""" + +import csv +import json +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +ROOT = Path(__file__).resolve().parent.parent +RESULTS = ROOT / "results" +CSV_MASTER = ROOT / "Interfaze Benchmarks - running.csv" +OUT = RESULTS / "gemini35flash_vs_deepseekv4flash.png" + +# Validated palette (dataviz references/palette.md), light mode. +SURFACE = "#fcfcfb" +GEMINI = "#2a78d6" # slot 1 +DEEPSEEK = "#eb6834" # slot 2 +INK = "#0b0b0b" +INK_MUTED = "#898781" +GRID = "#e1e0d9" +MONO = ["DejaVu Sans Mono", "Menlo", "monospace"] + +GEMINI_COL = "Gemini-3.5-flash" + +# Why DeepSeek has no number, verified by direct probe against the Fireworks +# endpoint rather than assumed from docs: +# image -> HTTP 400 "This model does not support image inputs" +# audio -> HTTP 400, content must be a plain string +UNSUPPORTED = { + "OCRBench V2": "no image input", + "olmOCR": "no image input", + "RefCoco": "no image input", + "MMMU-Pro": "no image input", + "VoxPoppuliCleaned-AA": "no audio input", + "TheSOB": "not in this repo", +} + +# Benchmark label -> our measured metrics file + extractor. +# NB: gpqa_fireworks' model_slug strips hyphens (deepseekv4flash) while the +# other runners keep them (deepseek-v4-flash) — inherited from gpqa_openrouter. +MEASURED = { + "Spider-2.0-lite": ( + "spider2_lite_local_fireworks_deepseek-v4-flash_metrics.json", + lambda m: m["accuracy_of_local_135"] * 100, + ), + "MMMLU": ( + "mmmlulite_fireworks_accounts-fireworks-models-deepseek-v4-flash_reasoningoff_metrics.json", + lambda m: m["macro_accuracy"] * 100, + ), + "GPQA-Diamond": ( + "fireworks_deepseekv4flash_thinkingoff_gpqa_diamond_metrics.json", + lambda m: m["accuracy"] * 100, + ), +} + +DISPLAY = { + "OCRBench V2": "OCRBench V2", + "olmOCR": "olmOCR", + "RefCoco": "RefCOCO", + "VoxPoppuliCleaned-AA": "VoxPopuli-AA", + "TheSOB": "TheSOB", + "Spider-2.0-lite": "Spider-2.0-Lite", + "GPQA-Diamond": "GPQA Diamond", + "MMMLU": "MMMLU", + "MMMU-Pro": "MMMU-Pro", +} + + +def load_gemini() -> dict: + """Pull the Gemini-3.5-flash column out of the team's master CSV.""" + out = {} + with open(CSV_MASTER, newline="", encoding="utf-8") as f: + for row in csv.DictReader(f): + bench = (row.get("Benchmark") or "").strip() + raw = (row.get(GEMINI_COL) or "").strip() + if not bench or not raw: + continue + try: + out[bench] = float(raw) + except ValueError: + continue # price rows, "Does not support ..." cells + return out + + +# A run in flight is its own state. Its partial accuracy is NOT usable as an +# estimate: mmmlu_multi walks languages in order, so a half-finished run covers +# ~7 of 14 languages and its macro average is biased by construction. +IN_FLIGHT = { + "MMMLU": ( + "mmmlulite_fireworks_accounts-fireworks-models-deepseek-v4-flash_reasoningoff_responses.jsonl", + 19950, + ), + "GPQA-Diamond": ( + "fireworks_deepseekv4flash_thinkingoff_gpqa_diamond_responses.jsonl", + 198, + ), +} + + +def load_deepseek() -> dict: + out = {} + for bench, (fname, pick) in MEASURED.items(): + p = RESULTS / fname + if p.exists(): + out[bench] = round(pick(json.loads(p.read_text())), 2) + return out + + +def pending_note(bench: str) -> str: + """'running (n/total)' for an unfinished run, else a plain not-run.""" + spec = IN_FLIGHT.get(bench) + if not spec: + return UNSUPPORTED.get(bench, "not run") + fname, total = spec + p = RESULTS / fname + done = sum(1 for _ in p.open()) if p.exists() else 0 + return f"running… {done:,}/{total:,}" + + +def main(): + gem = load_gemini() + ds = load_deepseek() + + # Comparable rows first — that is the actual comparison, so it reads first. + comparable = [b for b in DISPLAY if b in ds and b in gem] + others = [b for b in DISPLAY if b not in ds and b in gem] + order = comparable + others + + fig, ax = plt.subplots(figsize=(12.5, 7.6), facecolor=SURFACE) + ax.set_facecolor(SURFACE) + + h = 0.34 + gap = 0.04 # surface gap between the paired bars, not a border + ys = list(range(len(order)))[::-1] + + for y, bench in zip(ys, order): + g = gem[bench] + ax.barh(y + (h + gap) / 2, g, height=h, color=GEMINI, zorder=3) + ax.annotate( + f"{g:.1f}", + xy=(g, y + (h + gap) / 2), + xytext=(6, 0), + textcoords="offset points", + va="center", + fontsize=10, + fontfamily=MONO, + color=INK, + fontweight="bold", + ) + + if bench in ds: + d = ds[bench] + ax.barh(y - (h + gap) / 2, d, height=h, color=DEEPSEEK, zorder=3) + ax.annotate( + f"{d:.1f}", + xy=(d, y - (h + gap) / 2), + xytext=(6, 0), + textcoords="offset points", + va="center", + fontsize=10, + fontfamily=MONO, + color=INK, + fontweight="bold", + ) + else: + # No bar, and say why — an empty row must never read as a zero score. + ax.annotate( + f"— {pending_note(bench)}", + xy=(0, y - (h + gap) / 2), + xytext=(4, 0), + textcoords="offset points", + va="center", + fontsize=9.5, + fontfamily=MONO, + color=DEEPSEEK, + style="italic", + ) + + ax.set_yticks(ys) + ax.set_yticklabels( + [DISPLAY[b] for b in order], fontsize=11.5, fontfamily=MONO, color=INK + ) + ax.set_xlim(0, 108) + ax.set_xticks([0, 20, 40, 60, 80, 100]) + ax.set_xticklabels( + ["0", "20", "40", "60", "80", "100"], + fontsize=9.5, + fontfamily=MONO, + color=INK_MUTED, + ) + ax.xaxis.grid(True, color=GRID, linewidth=0.8, zorder=0) # solid hairline + ax.set_axisbelow(True) + for side in ("top", "right", "left"): + ax.spines[side].set_visible(False) + ax.spines["bottom"].set_color(GRID) + ax.tick_params(axis="y", length=0) + ax.tick_params(axis="x", length=0) + + # Separator under the comparable block — the rows below it are not a contest. + if comparable and others: + ax.axhline(len(order) - len(comparable) - 0.5, color=GRID, linewidth=1.2) + + ax.legend( + handles=[ + plt.Rectangle( + (0, 0), + 1, + 1, + color=GEMINI, + label=f"{GEMINI_COL} — from repo CSV (team's runs)", + ), + plt.Rectangle( + (0, 0), 1, 1, color=DEEPSEEK, label="DeepSeek-V4-Flash — MEASURED here" + ), + ], + loc="lower center", + bbox_to_anchor=(0.5, -0.175), + ncol=2, + frameon=False, + prop={"family": MONO, "size": 10}, + labelcolor=[GEMINI, DEEPSEEK], + ) + + fig.text( + 0.008, + 0.965, + "Gemini-3.5-flash vs DeepSeek-V4-Flash", + fontsize=15, + fontfamily=MONO, + color=INK, + ) + fig.text( + 0.008, + 0.925, + f"higher is better · only the {len(comparable)} row(s) above the line are a like-for-like comparison", + fontsize=10, + fontfamily=MONO, + color=INK_MUTED, + ) + fig.text( + 0.008, + 0.035, + "DeepSeek V4 Flash is text-only (probed: image -> HTTP 400 'does not support image inputs'; audio -> 400). Those rows have no bar, not a zero.\n" + "Thinking settings differ and are NOT harmonised: Gemini GPQA/MMMLU were run thinking ON; our DeepSeek runs use reasoning_effort=none (its floor).\n" + "Spider-2.0-Lite for DeepSeek is 27.4% after fixing an unclosed-markdown-fence bug in the SQL extractor; before the fix the same responses scored 15.6%.", + fontsize=8.5, + fontfamily=MONO, + color=INK_MUTED, + linespacing=1.7, + ) + + fig.subplots_adjust(left=0.16, right=0.97, top=0.88, bottom=0.26) + fig.savefig(OUT, dpi=200, facecolor=SURFACE) + print(f"wrote {OUT}\n") + print(f"{'benchmark':18} {GEMINI_COL:>18} {'deepseek-v4-flash':>18}") + for b in order: + d = f"{ds[b]:.2f}" if b in ds else f"n/a ({pending_note(b)})" + print(f" {DISPLAY[b]:16} {gem[b]:18.2f} {d:>18}") + + +if __name__ == "__main__": + main() diff --git a/scripts/plot_inkling_radar.py b/scripts/plot_inkling_radar.py new file mode 100644 index 0000000..2713456 --- /dev/null +++ b/scripts/plot_inkling_radar.py @@ -0,0 +1,380 @@ +""" +Radar chart of the full Inkling benchmark suite, with Inkling-Small alongside. + +Provenance is the whole point of this chart's design, so it is encoded three ways +(colour, line style, and a per-point tag) and never by colour alone: + + MEASURED — Inkling, run by this repo. Solid line, filled markers. + pub — Inkling-Small, Thinking Machines' published figure. Not run here. + est — Inkling-Small, ESTIMATED by inference from the published table. + Not a measurement of any kind. + +Inkling-Small is not on Fireworks serverless (dedicated deployment only), so +there is nothing measured to plot for it. + + uv run python scripts/plot_inkling_radar.py +""" + +import json +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +from matplotlib.lines import Line2D +from matplotlib.patches import Polygon + +ROOT = Path(__file__).resolve().parent.parent +RESULTS = ROOT / "results" +OUT = RESULTS / "inkling_benchmarks_radar.png" + +# Validated palette (references/palette.md), light mode. Both slots pass every +# check vs this surface (worst adjacent CVD dE 24.7). +SURFACE = "#fcfcfb" +MEASURED = "#2a78d6" # slot 1 — blue +CLAIMED = "#eb6834" # slot 2 — orange +INK = "#0b0b0b" +INK_MUTED = "#898781" +GRID = "#e1e0d9" +MONO = ["DejaVu Sans Mono", "Menlo", "monospace"] + +# Thinking Machines' published table (thinkingmachines.ai/news/inkling-small/), +# reported at effort=0.99 — MAX thinking, not the floor our runs used. +PUBLISHED_SMALL = {"GPQA Diamond": 88.3, "MMMU-Pro": 73.1} + +# Where nothing is published, an explicit estimate. Rationale from the published +# table's own shape: Inkling-Small holds parity on short-form reasoning and +# vision (GPQA +1.1, MMMU-Pro -0.4) but drops hard on factual recall (SimpleQA +# 20.9 vs 43.9) and long-horizon agentic work (Tau-3 Banking 13.6 vs 23.7, +# Terminal-Bench 52.7 vs 63.8). +# MMMLU — scaled by the published Global-MMLU-Lite ratio (86.8/88.7). +# VoxPopuli — audio is near parity (VoiceBench 90.0 vs 91.4, MMAU 77.5/77.2). +# OCR/RefCOCO — perception-bound like MMMU-Pro, so a small markdown. +# Spider — long unseen schema behaves like the agentic cluster: marked down. +ESTIMATE_RATIOS = { + "MMMLU": 86.8 / 88.7, + "VoxPopuli-AA": 0.994, + "OCRBench V2": 0.97, + "olmOCR": 0.97, + "RefCOCO": 0.96, + "Spider-2.0-Lite": 0.865, +} + + +def pct(x): + return round(x * 100, 2) + + +def load_measured(): + """Every axis with a COMPLETE measured run, north then clockwise.""" + + def j(name): + p = RESULTS / name + return json.loads(p.read_text()) if p.exists() else None + + ocr = j("ocrbench_v2_fireworks_inkling_metrics.json") + olm = j("olmocr_fireworks_inkling_metrics.json") + gpqa = j("fireworks_inkling_thinkingoff_gpqa_diamond_metrics.json") + mmmu = j( + "mmmupro_standard_fireworks_accounts-fireworks-models-inkling_reasoningoff_metrics.json" + ) + mmmlu = j( + "mmmlulite_fireworks_accounts-fireworks-models-inkling_reasoningoff_metrics.json" + ) + spider = j("spider2_lite_local_fireworks_inkling_metrics.json") + asr = j("voxpopuli_aa_fireworks_accounts_fireworks_models_inkling_metrics.json") + rc = j( + "refcoco_val_fireworks_accounts_fireworks_models_inkling_oracle_metrics.json" + ) + + rows = [ + ( + "OCRBench V2", + "Native OCR (EN)", + pct(ocr["en_overall"]) if ocr else None, + 10000 if ocr else 0, + ), + # olmOCR-bench writes its score to stdout, not a metrics file — see + # OLMOCR_SCORE below. + ("olmOCR", "Complex document processing", OLMOCR_SCORE, 1403), + ( + "RefCOCO", + "Object detection (oracle)", + pct(rc["accuracy"]) if rc else None, + rc["total"] if rc else 0, + ), + ( + "Spider-2.0-Lite", + "Text-to-SQL", + pct(spider["accuracy_of_local_135"]), + spider["total_local_subset"], + ), + ( + "VoxPopuli-AA", + "Speech recognition (1-WER)", + pct(1 - asr["corpus_wer"]), + asr.get("samples") or asr.get("num_samples"), + ), + ( + "MMMLU", + "Multilingual Q&A", + pct(mmmlu["macro_accuracy"]), + mmmlu["num_samples"], + ), + ( + "GPQA Diamond", + "PhD-level problem solving", + pct(gpqa["accuracy"]), + gpqa["total"], + ), + ( + "MMMU-Pro", + "Multimodal understanding (std)", + pct(mmmu["accuracy"]), + mmmu["num_samples"], + ), + ] + return [r for r in rows if r[2] is not None] + + +# olmOCR-bench prints its headline to stdout rather than persisting a metrics +# file, so it is pinned here from the run log (1403/1403 pages, 8413 tests). +OLMOCR_SCORE = 74.9 + + +def main(): + scores = load_measured() + labels = [s[0] for s in scores] + subs = [s[1] for s in scores] + vals = [s[2] for s in scores] + ns = [s[3] for s in scores] + n = len(scores) + + small_vals, small_tags = [], [] + for name, v in zip(labels, vals): + if name in PUBLISHED_SMALL: + small_vals.append(PUBLISHED_SMALL[name]) + small_tags.append("pub") + else: + small_vals.append(round(v * ESTIMATE_RATIOS[name], 1)) + small_tags.append("est") + + # Guard: a tag per value. If these ever diverge, zip() below would silently + # drop points rather than fail, which is how a mislabelled chart ships. + assert len(small_tags) == len(small_vals) == len(vals), ( + len(small_tags), + len(small_vals), + len(vals), + ) + + theta = [np.pi / 2 - 2 * np.pi * i / n for i in range(n)] + closed_theta = theta + [theta[0]] + + # No set_theta_zero_location — theta is computed absolutely above, so + # matplotlib's default (0 = east, CCW) keeps index 0 due north. + fig = plt.figure(figsize=(14.5, 10.5), facecolor=SURFACE) + ax = fig.add_subplot(111, polar=True, facecolor=SURFACE) + ax.set_ylim(0, 100) + ax.set_axis_off() # draw our own recessive web instead of matplotlib's + + # Web: solid hairline polygons, one shade off the surface (never dashed). + for r in (20, 40, 60, 80, 100): + ax.add_patch( + Polygon( + [(t, r) for t in theta], + closed=True, + fill=False, + edgecolor=GRID, + linewidth=0.8, + zorder=1, + ) + ) + for t in theta: + ax.plot([t, t], [0, 100], color=GRID, linewidth=0.8, zorder=1) + + # Inkling-Small first so the measured series sits on top of it. + ax.plot( + closed_theta, + small_vals + [small_vals[0]], + color=CLAIMED, + linewidth=2, + linestyle=(0, (5, 3)), + zorder=2, + ) + ax.scatter( + theta, + small_vals, + s=80, + facecolors=SURFACE, + edgecolors=CLAIMED, + linewidths=2, + zorder=3, + ) + + ax.fill(closed_theta, vals + [vals[0]], color=MEASURED, alpha=0.13, zorder=4) + ax.plot(closed_theta, vals + [vals[0]], color=MEASURED, linewidth=2, zorder=5) + ax.scatter( + theta, vals, s=90, color=MEASURED, edgecolors=SURFACE, linewidths=2, zorder=6 + ) + + # Direct value labels — every value readable without a tooltip. Offsets run + # along the spoke (outward for the higher series) so the pair stays apart + # even when the values nearly coincide. + for t, v, sv, tag in zip(theta, vals, small_vals, small_tags): + ux, uy = np.cos(t), np.sin(t) + m_dir, s_dir = (16, -18) if v >= sv else (-18, 16) + if abs(uy) >= abs(ux): + m_off, s_off = (ux * m_dir, uy * m_dir), (ux * s_dir, uy * s_dir) + else: + # Horizontal spokes: these labels are wide, so stack them + # perpendicular instead, nudged outward clear of the markers. + m_off, s_off = (ux * 10, 13), (ux * 10, -13) + ax.annotate( + f"{v:.1f}%", + xy=(t, v), + xytext=m_off, + textcoords="offset points", + ha="center", + va="center", + fontsize=10.5, + fontfamily=MONO, + color=MEASURED, + fontweight="bold", + zorder=7, + ) + ax.annotate( + f"{sv:.1f}% {tag}", + xy=(t, sv), + xytext=s_off, + textcoords="offset points", + ha="center", + va="center", + fontsize=9.5, + fontfamily=MONO, + color=CLAIMED, + zorder=7, + ) + + # One scale anchor on an empty diagonal — ring ticks along a spoke collide + # with that axis's label block, and every value is direct-labelled anyway. + ax.annotate( + "100", + xy=(np.pi / 2 - np.pi / n, 100), + xytext=(8, 4), + textcoords="offset points", + ha="left", + va="center", + fontsize=8.5, + fontfamily=MONO, + color=INK_MUTED, + zorder=5, + ) + + # Axis labels outside the web; descriptor and n on one line. + for t, name, sub, ncount in zip(theta, labels, subs, ns): + x, y = np.cos(t), np.sin(t) + ha = "center" if abs(x) < 0.3 else ("left" if x > 0 else "right") + ax.annotate( + name, + xy=(t, 100), + xytext=(x * 26, y * 26 + 7), + textcoords="offset points", + ha=ha, + va="center", + fontsize=12.5, + fontfamily=MONO, + color=INK, + zorder=5, + ) + ax.annotate( + f"{sub} · n={ncount}", + xy=(t, 100), + xytext=(x * 26, y * 26 - 7), + textcoords="offset points", + ha=ha, + va="center", + fontsize=9, + fontfamily=MONO, + color=INK_MUTED, + zorder=5, + ) + + fig.text( + 0.5, + 0.973, + "Inkling vs Inkling-Small — complete Fireworks benchmark suite", + ha="center", + fontsize=15, + fontfamily=MONO, + color=INK, + ) + fig.text( + 0.5, + 0.945, + "higher is better on every axis · rings every 20 points · n = measured run's sample count", + ha="center", + fontsize=9.5, + fontfamily=MONO, + color=INK_MUTED, + ) + + fig.legend( + handles=[ + Line2D( + [], + [], + color=MEASURED, + lw=2, + marker="o", + markersize=8, + markeredgecolor=SURFACE, + label="Inkling — MEASURED here (reasoning_effort=none, the floor)", + ), + Line2D( + [], + [], + color=CLAIMED, + lw=2, + linestyle=(0, (5, 3)), + marker="o", + markersize=8, + markerfacecolor=SURFACE, + markeredgecolor=CLAIMED, + label="Inkling-Small — NOT measured: pub = published, est = estimated", + ), + ], + loc="upper center", + bbox_to_anchor=(0.5, 0.925), + frameon=False, + prop={"family": MONO, "size": 10.5}, + labelcolor=[MEASURED, CLAIMED], + handlelength=3, + borderaxespad=0, + ) + + fig.text( + 0.5, + 0.045, + "Inkling-Small is not on Fireworks serverless (dedicated deployment only) — nothing is measured for it here.\n" + "pub = published at effort=0.99 (MAX thinking), NOT comparable to our floor runs · est = our inference, 6 of 8 axes, treat as hypotheses.\n" + "RefCOCO is the format-tolerant oracle re-score (strict single-parser score is 32.4% — Inkling emits 4+ coordinate conventions).\n" + "MMMU-Pro is the standard setting, the only one published for both (our vision run measured 68.1%; composite 70.7%).", + ha="center", + fontsize=8.5, + fontfamily=MONO, + color=INK_MUTED, + linespacing=1.8, + ) + + fig.subplots_adjust(left=0.23, right=0.77, top=0.80, bottom=0.185) + fig.savefig(OUT, dpi=200, facecolor=SURFACE) + print(f"wrote {OUT}\n") + print(f"{'benchmark':18} {'inkling':>10} {'small':>10} n") + for (name, _s, v, ncount), sv, tag in zip(scores, small_vals, small_tags): + print(f" {name:16} {v:9.2f}% {sv:9.1f}% {tag:4} n={ncount}") + + +if __name__ == "__main__": + main() diff --git a/scripts/report_scores.py b/scripts/report_scores.py new file mode 100644 index 0000000..5fe27a7 --- /dev/null +++ b/scripts/report_scores.py @@ -0,0 +1,537 @@ +""" +Detailed benchmark scores for every model with results on disk. + + uv run python scripts/report_scores.py # interactive picker + uv run python scripts/report_scores.py --all # every model + uv run python scripts/report_scores.py --model inkling + uv run python scripts/report_scores.py --list # just the model names + uv run python scripts/report_scores.py --model inkling --tsv # paste into Sheets + +Reads results/*_metrics.json plus the olmOCR run logs (olmOCR-bench prints its +per-split table to stdout instead of persisting metrics, so the logs are the +only source for those numbers). + +Model names are canonicalised across providers — accounts/fireworks/models/inkling, +thinkingmachines/inkling and google/gemini-3.7-flash all collapse to the bare +model name, so one row means one model regardless of which host produced it. +Where the same model+benchmark exists from two hosts, both are kept and the +provider is shown, because the hosts are NOT equivalent at the same nominal +reasoning setting. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +RESULTS = ROOT / "results" +LOGS = ROOT / "logs" + +# --------------------------------------------------------------------------- +# Column orders, exactly as the sheet expects them +# --------------------------------------------------------------------------- + +OCR_COLS = [ + ("Avg Real", "en_overall"), + ("Recog", "text_recognition"), + ("Refer", "text_detection"), + ("Spot", "text_spotting"), + ("Extract", "relationship_extraction"), + ("Parse", "element_parsing"), + ("Calc", "mathematical_calculation"), + ("Understand", "visual_text_understanding"), + ("Reason", "knowledge_reasoning"), +] + +OLMOCR_COLS = [ + ("Overall", None), + ("ArXiv", "arxiv_math"), + ("OldScansMath", "old_scans_math"), + ("Tables", "table_tests"), + ("OldScans", "old_scans"), + ("Headers", "headers_footers"), + ("MultiCol", "multi_column"), + ("LongTinyText", "long_tiny_text"), + ("Base", "baseline"), + ("Real Overall", None), +] + +GPQA_COLS = [ + ("Overall", None), + ("Physics (n=86)", "Physics"), + ("Chemistry (n=93)", "Chemistry"), + ("Biology (n=19)", "Biology"), +] + +MMMLU_LANGS = [ + "FR_FR", + "PT_BR", + "BN_BD", + "JA_JP", + "DE_DE", + "YO_NG", + "ES_LA", + "ID_ID", + "ZH_CN", + "SW_KE", + "IT_IT", + "AR_XY", + "KO_KR", + "HI_IN", +] + +MMMU_STD_SUBJECTS = """overall Art Electronics Economics Marketing Finance Art_Theory +Public_Health Basic_Medical_Science Sociology Literature Physics Energy_and_Power History +Design Materials Biology Psychology Geography Manage Pharmacy Agriculture Clinical_Medicine +Accounting Computer_Science Architecture_and_Engineering Chemistry Math +Diagnostics_and_Laboratory_Medicine Mechanical_Engineering Music""".split() + +MMMU_VIS_SUBJECTS = """overall Economics Art_Theory Basic_Medical_Science Art Literature +Pharmacy Clinical_Medicine Sociology Public_Health Design Physics History Chemistry +Electronics Marketing Geography Math Biology Computer_Science Manage Finance Agriculture +Accounting Psychology Mechanical_Engineering Diagnostics_and_Laboratory_Medicine +Energy_and_Power Materials Architecture_and_Engineering Music""".split() + + +# --------------------------------------------------------------------------- +# Loading +# --------------------------------------------------------------------------- + + +def canon(model: str) -> str: + """Bare model name, provider prefixes and slug mangling removed.""" + m = model.strip().rsplit("/", 1)[-1].lower() + m = re.sub(r"^accounts-fireworks-models-", "", m) + # filename slugs turn dots into dashes: gemini-3-7-flash -> gemini-3.7-flash + m = re.sub(r"^gemini-(\d)-(\d)-", r"gemini-\1.\2-", m) + m = re.sub(r"^inklingsmall$", "inkling-small", m) + m = re.sub(r"^thinkingmachinesinklingsmall$", "inkling-small", m) + m = re.sub(r"^gemini37flash$", "gemini-3.7-flash", m) + return m + + +def _pct(x): + return None if x is None else round(float(x) * 100, 2) + + +def load_metrics() -> dict: + """{model: {benchmark_key: payload}} from results/*_metrics.json.""" + out: dict[str, dict] = {} + + def _n(payload): + """Sample count, for choosing between duplicate runs of the same + benchmark. Different hosts write different filenames for the same model + (voxpopuli_aa_gemini_* vs voxpopuli_aa_openrouter_google_*), and a 2-row + smoke test must never shadow a full run just because its filename sorts + later.""" + for k in ("n", "num_samples", "total", "total_local_subset"): + v = payload.get(k) + if isinstance(v, int): + return v + return 0 + + def put(model, key, payload, source=""): + bucket = out.setdefault(canon(model), {}) + prev = bucket.get(key) + payload = dict(payload, source=source) + if prev is None or _n(payload) >= _n(prev): + if prev is not None and _n(prev): + payload["shadowed"] = (prev.get("source"), _n(prev)) + bucket[key] = payload + else: + prev.setdefault("shadowed", (source, _n(payload))) + + for p in sorted(RESULTS.glob("*_metrics.json")): + name = p.name + try: + d = json.loads(p.read_text()) + except json.JSONDecodeError: + continue + model = d.get("model") or "" + + if name.startswith("ocrbench_v2_"): + model = model or name.replace("ocrbench_v2_", "").replace( + "_metrics.json", "" + ) + covered = sum(1 for v in d.get("en_scores", {}).values() if v.get("count")) + put( + model, + "ocrbench", + { + "en_overall": _pct(d.get("en_overall")), + "cn_overall": _pct(d.get("cn_overall")), + "cats": { + k: _pct(v["avg"]) if v.get("count") else None + for k, v in d.get("en_scores", {}).items() + }, + "counts": { + k: v.get("count", 0) for k, v in d.get("en_scores", {}).items() + }, + "partial": covered < 8, + "covered": covered, + }, + ) + elif name.startswith("mmmupro_"): + setting = "standard" if "_standard_" in name else "vision" + reasoning = "high" if "reasoninghigh" in name else "off" + put( + model, + f"mmmupro_{setting}_{reasoning}", + { + "overall": _pct(d.get("accuracy")), + "n": d.get("num_samples"), + "subjects": { + k: _pct(v["accuracy"]) + for k, v in (d.get("per_subject") or {}).items() + }, + "reasoning": reasoning, + }, + ) + elif name.startswith("mmmlulite_"): + put( + model, + "mmmlu", + { + "macro": _pct(d.get("macro_accuracy")), + "n": d.get("num_samples"), + "langs": { + k: _pct(v["accuracy"]) + for k, v in (d.get("per_language") or {}).items() + }, + "reasoning": "high" if "reasoninghigh" in name else "off", + }, + ) + elif name.startswith("refcoco_"): + split = name.split("_")[1] + oracle = "_oracle_" in name + put( + model, + f"refcoco_{split}" + ("_oracle" if oracle else ""), + { + "acc": _pct(d.get("accuracy")), + "mean_iou": round(d.get("mean_iou", 0), 4), + "n": d.get("total"), + "split": split, + }, + ) + elif name.startswith("voxpopuli_aa_"): + wer = d.get("corpus_wer") + put( + model, + "asr", + { + "wer": _pct(wer), + "inv": _pct(1 - wer) if wer is not None else None, + "cer": _pct(d.get("corpus_cer")), + "n": d.get("num_samples"), + }, + ) + elif name.startswith("spider2_lite_local_"): + put( + model, + "spider2", + { + "acc": _pct(d.get("accuracy_of_local_135")), + "correct": d.get("correct"), + "n": d.get("total_local_subset"), + }, + ) + elif "gpqa_diamond" in name: + model = model or name.split("_")[1] + put( + model, + "gpqa", + { + "overall": _pct(d.get("accuracy")), + "n": d.get("total"), + "domains": { + k: _pct(v["accuracy"]) + for k, v in (d.get("per_domain") or {}).items() + }, + }, + ) + return out + + +def load_olmocr() -> dict: + """olmOCR-bench prints its table to stdout, so parse the run logs.""" + out: dict[str, dict] = {} + # olmOCR-bench prints its table to stdout and persists nothing, so scan any + # captured output: per-step suite logs, standalone runs, or a pasted summary. + candidates = [] + for pat in ("**/olmocr*.log", "**/*olmocr*.txt", "**/suite*.log"): + candidates += sorted(LOGS.glob(pat)) + for log in dict.fromkeys(candidates): + text = log.read_text(errors="ignore").replace("\r", "\n") + head = re.search( + r"^(\S+)\s*:\s*Average Score:\s*([\d.]+)%\s*±\s*([\d.]+)%", text, re.M + ) + if not head: + continue + model, overall, ci = head.group(1), float(head.group(2)), float(head.group(3)) + splits = { + m.group(1): float(m.group(2)) + for m in re.finditer(r"^\s+(\w+)\.jsonl\s*:\s*([\d.]+)%", text, re.M) + } + base = re.search(r"^\s+baseline\s*:\s*([\d.]+)%", text, re.M) + if base: + splits["baseline"] = float(base.group(1)) + out[canon(model)] = { + "overall": overall, + "ci": ci, + "splits": splits, + "source": str(log.relative_to(ROOT)), + } + return out + + +# --------------------------------------------------------------------------- +# Rendering +# --------------------------------------------------------------------------- + + +def fmt(v, width=7): + return f"{v:>{width}.2f}" if isinstance(v, (int, float)) else f"{'-':>{width}}" + + +def table(headers, rows, tsv=False): + if tsv: + print("\t".join(headers)) + for r in rows: + print( + "\t".join( + "" + if c is None + else (f"{c:.2f}" if isinstance(c, float) else str(c)) + for c in r + ) + ) + return + widths = ( + [ + max( + len(str(h)), + *( + len(f"{c:.2f}") + if isinstance(c, float) + else len(str(c if c is not None else "-")) + for c in col + ), + ) + for h, col in zip(headers, zip(*rows)) + ] + if rows + else [len(h) for h in headers] + ) + print( + " ".join( + h.ljust(w) if i == 0 else h.rjust(w) + for i, (h, w) in enumerate(zip(headers, widths)) + ) + ) + print(" ".join("-" * w for w in widths)) + for r in rows: + cells = [] + for i, (c, w) in enumerate(zip(r, widths)): + s = f"{c:.2f}" if isinstance(c, float) else str(c if c is not None else "-") + cells.append(s.ljust(w) if i == 0 else s.rjust(w)) + print(" ".join(cells)) + + +def report(models: list[str], data: dict, olm: dict, tsv=False): + def get(model, key): + return data.get(model, {}).get(key) + + # ---- OCRBench V2 + rows, notes = [], [] + for m in models: + d = get(m, "ocrbench") + if not d: + continue + row = [m, d["en_overall"]] + [d["cats"].get(k) for _, k in OCR_COLS[1:]] + rows.append(row) + if d["partial"]: + missing = [k for k, c in d["counts"].items() if not c] + notes.append( + f" ! {m}: PARTIAL — only {d['covered']}/8 EN categories have samples " + f"(missing: {', '.join(missing)}). 'Avg Real' averages the covered ones, " + f"so it is NOT comparable to a full run." + ) + if rows: + print("\n=== OCRBench V2 (EN) ===") + table([c for c, _ in [("model", None)] + OCR_COLS], rows, tsv) + for n in notes: + print(n) + + # ---- olmOCR + rows = [] + for m in models: + d = olm.get(m) + if not d: + continue + row = [m, d["overall"]] + [d["splits"].get(k) for _, k in OLMOCR_COLS[1:-1]] + row.append(f"{d['overall']:.1f}±{d['ci']:.1f}") + rows.append(row) + if rows: + print("\n=== olmOCR-bench ===") + table([c for c, _ in [("model", None)] + OLMOCR_COLS], rows, tsv) + + # ---- RefCOCO (every split found, so a val number is never mistaken for TestA) + rows = [] + for m in models: + for key, d in sorted((data.get(m) or {}).items()): + if not key.startswith("refcoco"): + continue + label = ( + f"{d['split']}{' (oracle)' if key.endswith('_oracle') else ' (strict)'}" + ) + rows.append([m, label, d["acc"], d["mean_iou"], d["n"]]) + if rows: + print("\n=== RefCOCO ===") + table(["Model", "split/scoring", "Acc@0.5", "mean IoU", "n"], rows, tsv) + + # ---- ASR + rows = [ + [ + m, + get(m, "asr")["inv"], + get(m, "asr")["wer"], + get(m, "asr")["cer"], + get(m, "asr")["n"], + ] + for m in models + if get(m, "asr") + ] + if rows: + print("\n=== ASR (VoxPopuliCleaned-AA) ===") + table(["Model", "1-WER", "WER", "CER", "n"], rows, tsv) + print( + " (sheet convention is 1-WER; WER shown too since lower-is-better there)" + ) + + # ---- GPQA + rows = [] + for m in models: + d = get(m, "gpqa") + if d: + rows.append( + [m, d["overall"]] + [d["domains"].get(k) for _, k in GPQA_COLS[1:]] + ) + if rows: + print("\n=== GPQA Diamond ===") + table([c for c, _ in [("Model", None)] + GPQA_COLS], rows, tsv) + + # ---- MMMLU + rows = [] + for m in models: + d = get(m, "mmmlu") + if d: + rows.append([m, d["macro"]] + [d["langs"].get(lg) for lg in MMMLU_LANGS]) + if rows: + print("\n=== MMMLU ===") + table(["Model", "macro"] + MMMLU_LANGS, rows, tsv) + + # ---- MMMU-Pro, subjects as rows in the sheet's order + for setting, subjects in ( + ("standard", MMMU_STD_SUBJECTS), + ("vision", MMMU_VIS_SUBJECTS), + ): + cols, found = [], [] + for m in models: + for reasoning in ("off", "high"): + d = get(m, f"mmmupro_{setting}_{reasoning}") + if d: + cols.append((f"{m} ({reasoning})", d)) + found.append(m) + if not cols: + continue + print( + f"\n=== MMMU-Pro-{'Standard Split' if setting == 'standard' else 'Vision'} ===" + ) + rows = [] + for s in subjects: + row = [s] + for _, d in cols: + row.append(d["overall"] if s == "overall" else d["subjects"].get(s)) + rows.append(row) + table(["Subject"] + [c for c, _ in cols], rows, tsv) + + # ---- Spider2 + rows = [ + [ + m, + get(m, "spider2")["acc"], + f"{get(m, 'spider2')['correct']}/{get(m, 'spider2')['n']}", + ] + for m in models + if get(m, "spider2") + ] + if rows: + print("\n=== Spider-2.0-lite (SQLite subset) ===") + table(["Model", "Spider-2.0-lite", "correct"], rows, tsv) + for m in models: + d = get(m, "spider2") + if d and d["n"] and d["correct"] is not None and d["n"] < 135: + print(f" ! {m}: only {d['n']} examples scored — not a full run") + + +def main(): + ap = argparse.ArgumentParser(description="Detailed benchmark scores from results/") + ap.add_argument("--model", action="append", help="model name (repeatable)") + ap.add_argument("--all", action="store_true", help="every model found") + ap.add_argument("--list", action="store_true", help="list model names and exit") + ap.add_argument( + "--tsv", action="store_true", help="tab-separated, for pasting into Sheets" + ) + args = ap.parse_args() + + data, olm = load_metrics(), load_olmocr() + known = sorted(set(data) | set(olm)) + if not known: + sys.exit(f"No results found in {RESULTS}") + + if args.list: + for m in known: + n = len(data.get(m, {})) + (1 if m in olm else 0) + print(f"{m:24} {n} benchmark result(s)") + return + + if args.all: + chosen = known + elif args.model: + chosen = [] + for want in args.model: + hits = [m for m in known if want.lower() in m] + if not hits: + sys.exit(f"No model matching {want!r}. Known: {', '.join(known)}") + chosen += hits + else: + # Interactive by default. + print("Models with results:\n") + for i, m in enumerate(known, 1): + n = len(data.get(m, {})) + (1 if m in olm else 0) + print(f" {i:2}. {m:24} ({n} result(s))") + print(f" {len(known) + 1:2}. ALL") + try: + raw = input("\nPick a number (or name, blank = ALL): ").strip() + except EOFError: + raw = "" + if not raw or raw == str(len(known) + 1): + chosen = known + elif raw.isdigit() and 1 <= int(raw) <= len(known): + chosen = [known[int(raw) - 1]] + else: + chosen = [m for m in known if raw.lower() in m] or sys.exit( + f"No match for {raw!r}" + ) + + print(f"\nModels: {', '.join(chosen)}") + report(chosen, data, olm, tsv=args.tsv) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_fireworks_suite.sh b/scripts/run_fireworks_suite.sh new file mode 100755 index 0000000..7e05c1f --- /dev/null +++ b/scripts/run_fireworks_suite.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Run the whole benchmark suite against one Fireworks-hosted model. +# +# Serialized on purpose: every runner shares one Fireworks account rate limit, +# so running them concurrently just burns retries on 429s. +# +# Usage: +# scripts/run_fireworks_suite.sh # inkling (serverless) +# MODEL=accounts/fireworks/models/inkling-small scripts/run_fireworks_suite.sh +# ONLY=ocrbench,mmmlu scripts/run_fireworks_suite.sh # subset +# SKIP=ocrbench,olmocr scripts/run_fireworks_suite.sh # all but these +# +# The two OCR benchmarks dominate the cost of a full pass (OCRBench v2 is 10,000 +# image samples, olmOCR-bench 1,403 PDF pages), which is why they're separately +# skippable. +# +# Every runner checkpoints, so re-running skips completed samples. Logs land in +# logs/fireworks_/.log. + +set -uo pipefail +cd "$(dirname "$0")/.." + +MODEL="${MODEL:-accounts/fireworks/models/inkling}" +# PROVIDER lets the same suite target another host (e.g. OpenRouter, which is +# the only place inkling-small is reachable). REASONING is the effort passed +# through; note "none" is a floor on Fireworks but a true off on OpenRouter. +PROVIDER="${PROVIDER:-fireworks}" +REASONING="${REASONING:-off}" +export BENCH_OLMOCR_PROVIDER="$PROVIDER" +export BENCH_OPENROUTER_EFFORT="${OPENROUTER_EFFORT:-none}" +export BENCH_OLMOCR_EFFORT="$BENCH_OPENROUTER_EFFORT" +# ASR effort is pinned separately: inkling wants "none" (at "high" it returns +# content=None after ~233s/sample), but some models REJECT "none" outright — +# gemini-3.7-flash 400s with "Reasoning is mandatory ... cannot be disabled". +ASR_EFFORT="${ASR_EFFORT:-none}" +SLUG="$(printf '%s' "${MODEL##*/}" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9' '-' | sed 's/-$//')" +LOG_DIR="logs/${PROVIDER}_${SLUG}" +mkdir -p "$LOG_DIR" + +# RefCOCO split: val is what refcoco.py (the interfaze reference run) uses. +REFCOCO_SPLIT="${REFCOCO_SPLIT:-val}" +ONLY="${ONLY:-}" +SKIP="${SKIP:-}" + +run_step() { + local name="$1"; shift + if [ -n "$ONLY" ] && [[ ",$ONLY," != *",$name,"* ]]; then + echo "[skip] $name (not in ONLY)" + return 0 + fi + if [ -n "$SKIP" ] && [[ ",$SKIP," == *",$name,"* ]]; then + echo "[skip] $name (in SKIP)" + return 0 + fi + local log="$LOG_DIR/$name.log" + echo "=== [$(date +%H:%M:%S)] $name -> $log" + if "$@" >"$log" 2>&1; then + echo "=== [$(date +%H:%M:%S)] $name OK" + else + # Keep going — one benchmark failing shouldn't strand the other eight. + echo "=== [$(date +%H:%M:%S)] $name FAILED (exit $?), see $log" + fi +} + +# Cheapest first so partial results show up early. +# ASR is pinned to effort=none even when the rest of the suite runs higher. +# Measured on inkling-small via OpenRouter: effort=high returns content=None +# after ~233s per sample (every sample would burn 3 retries and fail), while +# effort=none answers in ~1.3s. Transcription gains nothing from thinking. +BENCH_OPENROUTER_EFFORT="$ASR_EFFORT" \ +run_step asr uv run -m benchmarks.asr.voxpopuli_aa_multi \ + --provider "$PROVIDER" --model "$MODEL" + +run_step spider2 uv run -m benchmarks.spider2_lite.spider2_lite \ + --provider "$PROVIDER" --model "$MODEL" + +run_step mmmupro_standard uv run -m benchmarks.mmmu_pro.mmmu_pro_multi \ + --provider "$PROVIDER" --model "$MODEL" --setting standard --reasoning "$REASONING" + +run_step mmmupro_vision uv run -m benchmarks.mmmu_pro.mmmu_pro_multi \ + --provider "$PROVIDER" --model "$MODEL" --setting vision --reasoning "$REASONING" + +run_step refcoco uv run -m benchmarks.obj_detection.refcoco_multi \ + --provider "$PROVIDER" --model "$MODEL" --split "$REFCOCO_SPLIT" + +run_step olmocr uv run -m benchmarks.olmocr.olmocr_bench_fireworks --model "$MODEL" + +run_step mmmlu uv run -m benchmarks.mmmlu.mmmlu_multi \ + --provider "$PROVIDER" --model "$MODEL" --reasoning "$REASONING" + +run_step ocrbench uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks --model "$MODEL" \ + --provider "$PROVIDER" --reasoning "$BENCH_OPENROUTER_EFFORT" + +# Gated dataset — needs `hf auth login` plus accepted terms on the Hub. +# GPQA has separate per-host runners; gpqa_fireworks talks only to Fireworks, so +# an OpenRouter model id would 404 there. +if [ "$PROVIDER" = "openrouter" ]; then + run_step gpqa uv run -m benchmarks.gpqa.gpqa_openrouter --model "$MODEL" \ + --thinking "$([ "$REASONING" = "high" ] && echo on || echo off)" +else + run_step gpqa uv run -m benchmarks.gpqa.gpqa_fireworks --model "$MODEL" +fi + +# Grounding is scored strictly above; this adds the repo's uniform +# format-tolerant oracle pass (upper bound) for every RefCOCO run present. +run_step refcoco_oracle uv run -m benchmarks.obj_detection.reeval_format_tolerant + +echo "=== [$(date +%H:%M:%S)] suite finished for $MODEL" diff --git a/scripts/setup_bfcl.sh b/scripts/setup_bfcl.sh new file mode 100755 index 0000000..bd2fa91 --- /dev/null +++ b/scripts/setup_bfcl.sh @@ -0,0 +1,251 @@ +#!/usr/bin/env bash +# Set up the Berkeley Function Calling Leaderboard (BFCL) for Fireworks-hosted +# models, idempotently. +# +# scripts/setup_bfcl.sh +# +# Why a script and not a one-off: BFCL lives in a third-party clone under +# external/ (gitignored), so any edit made directly in that tree is lost on +# re-clone or a fresh checkout. This script IS the durable artifact — re-running +# it reproduces the whole setup, including the model registrations. +# +# It does four things: +# 1. clones gorilla (shallow) if absent +# 2. builds a SEPARATE venv — BFCL's pins conflict with this repo's +# (olmocr==0.4.27 and friends), so it must not share .venv +# 3. writes BFCL's own .env with FIREWORKS_API_KEY *uppercase*: BFCL's +# FireworksHandler does a bare os.getenv("FIREWORKS_API_KEY") with no +# fallback, and this project's .env spells the key lowercase +# 4. registers the Fireworks models in model_config.py, append-if-absent +# +# After it runs: +# external/bfcl-venv/bin/bfcl generate --model \ +# --test-category simple --num-threads 4 +# external/bfcl-venv/bin/bfcl evaluate --model --test-category simple + +set -euo pipefail +cd "$(dirname "$0")/.." +ROOT="$PWD" + +GORILLA_DIR="$ROOT/external/gorilla" +BFCL_DIR="$GORILLA_DIR/berkeley-function-call-leaderboard" +VENV="$ROOT/external/bfcl-venv" +RUNS="$ROOT/external/bfcl-runs" +CONFIG="$BFCL_DIR/bfcl_eval/constants/model_config.py" + +# ---------------------------------------------------------------- 1. clone +if [ ! -d "$BFCL_DIR" ]; then + echo "[clone] ShishirPatil/gorilla (depth 1)" + mkdir -p "$ROOT/external" + git clone --depth 1 https://github.com/ShishirPatil/gorilla.git "$GORILLA_DIR" +else + echo "[skip] gorilla already cloned at $GORILLA_DIR" +fi + +# ---------------------------------------------------------------- 2. venv +if [ ! -x "$VENV/bin/bfcl" ]; then + echo "[venv] creating $VENV and installing bfcl_eval (editable)" + uv venv "$VENV" --python 3.12 >/dev/null + VIRTUAL_ENV="$VENV" uv pip install --python "$VENV/bin/python" -e "$BFCL_DIR" >/dev/null + # soundfile is an undeclared transitive dep: model_config.py imports every + # handler eagerly, including qwen.py -> qwen_agent -> soundfile. Without it, + # merely LOADING the model registry raises ModuleNotFoundError, so no model + # can run — not just the Qwen-agent ones. + VIRTUAL_ENV="$VENV" uv pip install --python "$VENV/bin/python" soundfile >/dev/null +else + echo "[skip] bfcl already installed in $VENV" +fi + +# ---------------------------------------------------------------- 3. env +mkdir -p "$RUNS" +# Accept either casing from the project .env, write the uppercase name BFCL wants. +FW_KEY="$( + grep -iE '^[[:space:]]*fireworks_api_key[[:space:]]*=' "$ROOT/.env" 2>/dev/null \ + | head -1 | cut -d= -f2- | tr -d ' "'"'" || true +)" +if [ -z "${FW_KEY:-}" ]; then + echo "[warn] no fireworks_api_key found in $ROOT/.env — BFCL will 401 until you add it" +fi +OR_KEY="$( + grep -iE '^[[:space:]]*openrouter_api_key[[:space:]]*=' "$ROOT/.env" 2>/dev/null \ + | head -1 | cut -d= -f2- | tr -d ' "'"'" || true +)" +BFCL_ENV="$RUNS/.env" +{ + echo "# generated by scripts/setup_bfcl.sh — do not edit by hand" + echo "FIREWORKS_API_KEY=${FW_KEY:-}" + echo "BFCL_PROJECT_ROOT=$RUNS" + # FireworksHandler.__init__ calls super().__init__() first, and + # OpenAICompletionsHandler builds an OpenAI() client that raises without + # OPENAI_API_KEY — before the Fireworks client replaces it. So a Fireworks run + # cannot start without *some* value here. It is never used for inference. + echo "OPENAI_API_KEY=placeholder-unused-by-fireworks-handler" + echo "OPENROUTER_API_KEY=${OR_KEY:-}" +} > "$BFCL_ENV" +cp "$BFCL_ENV" "$BFCL_DIR/.env" +echo "[env] wrote $BFCL_ENV (and a copy at $BFCL_DIR/.env)" + +# ------------------------------------------------- 4a. OpenRouter handler +# BFCL ships no OpenRouter handler. OpenRouter is OpenAI-compatible, so this is +# a minimal subclass that swaps in the client — it deliberately keeps the +# parent's OPENAI_COMPLETIONS model_style, which already emits the standard +# {"type": "function", ...} tool shape OpenRouter expects. +OR_HANDLER="$BFCL_DIR/bfcl_eval/model_handler/api_inference/openrouter.py" +if [ ! -f "$OR_HANDLER" ]; then + cat > "$OR_HANDLER" <<'PY' +import os + +from openai import OpenAI + +from bfcl_eval.model_handler.api_inference.openai_completion import ( + OpenAICompletionsHandler, +) + + +class OpenRouterHandler(OpenAICompletionsHandler): + """Any OpenRouter-hosted model over its OpenAI-compatible endpoint. + + Added by this repo's scripts/setup_bfcl.sh — not part of upstream BFCL. + Note the parent __init__ still builds an OpenAI() client first and raises + without OPENAI_API_KEY, hence the placeholder written into BFCL's .env. + """ + + def __init__(self, model_name, temperature, registry_name, is_fc_model, **kwargs) -> None: + super().__init__(model_name, temperature, registry_name, is_fc_model, **kwargs) + self.client = OpenAI( + base_url="https://openrouter.ai/api/v1", + api_key=os.getenv("OPENROUTER_API_KEY"), + ) +PY + echo "[handler] wrote $OR_HANDLER" +else + echo "[skip] OpenRouter handler already present" +fi + +# Make sure model_config.py imports it (idempotent). +python3 - "$CONFIG" <<'PY' +import sys +from pathlib import Path +p = Path(sys.argv[1]); src = p.read_text() +imp = ("from bfcl_eval.model_handler.api_inference.openrouter import (\n" + " OpenRouterHandler,\n)\n") +if "OpenRouterHandler" not in src: + anchor = "from bfcl_eval.model_handler.api_inference.fireworks import FireworksHandler\n" + assert anchor in src, "fireworks import anchor not found" + src = src.replace(anchor, anchor + imp, 1) + p.write_text(src) + print("[register] added OpenRouterHandler import") +else: + print("[skip] OpenRouterHandler already imported") +PY + +# ---------------------------------------------------------------- 4. register +# One entry per model, appended into api_inference_model_map only if its registry +# name is not already present. +# +# is_fc_model: NOT verifiable while both models 404. Registered in prompt mode +# (is_fc_model=False) as the safer default — an FC registration against a model +# that silently ignores `tools=` yields a near-zero score that reads like a +# capability result. Re-confirm on the first real run and flip to the -FC entry +# if the model does emit native tool calls. +python3 - "$CONFIG" <<'PY' +import sys, re +from pathlib import Path + +path = Path(sys.argv[1]) +src = path.read_text() + +ENTRIES = { + # registry name -> (model id, display name, org, license, is_fc, underscore_to_dot) + # + # underscore_to_dot: True when the model returns sanitised function names + # (math_factorial) instead of the dotted originals (math.factorial). This is + # provider-specific, not cosmetic: with the wrong value the checker rejects + # CORRECT calls as wrong_func_name. Verified empirically per provider — + # OpenRouter sanitises (OpenAI tool schemas forbid "." in names), Fireworks + # passes dots through. + "fireworks-qwen3p5-35b-a3b": ( + "accounts/fireworks/models/qwen3p5-35b-a3b", + "Qwen3.5-35B-A3B (Fireworks, prompt)", "Qwen", "apache-2.0", False, False, + ), + "fireworks-qwen3p5-35b-a3b-FC": ( + "accounts/fireworks/models/qwen3p5-35b-a3b", + "Qwen3.5-35B-A3B (Fireworks, FC)", "Qwen", "apache-2.0", True, False, + ), + "fireworks-gemma-4-31b-it": ( + "accounts/fireworks/models/gemma-4-31b-it", + "Gemma-4-31B-IT (Fireworks, prompt)", "Google", "gemma-terms-of-use", False, False, + ), + "fireworks-gemma-4-31b-it-FC": ( + "accounts/fireworks/models/gemma-4-31b-it", + "Gemma-4-31B-IT (Fireworks, FC)", "Google", "gemma-terms-of-use", True, False, + ), + # --- OpenRouter: both models ARE available here, unlike Fireworks serverless. + # is_fc_model=True is probe-verified: each emits native tool_calls when given + # `tools`, and OpenRouter advertises tools in supported_parameters for both. + "openrouter-qwen3.5-35b-a3b-FC": ( + "qwen/qwen3.5-35b-a3b", + "Qwen3.5-35B-A3B (OpenRouter, FC)", "Qwen", "apache-2.0", True, True, + ), + "openrouter-gemma-4-31b-it-FC": ( + "google/gemma-4-31b-it", + "Gemma-4-31B-IT (OpenRouter, FC)", "Google", "gemma-terms-of-use", True, True, + ), + # An available serverless model, used only to validate the plumbing. + "fireworks-qwen3p7-plus-FC": ( + "accounts/fireworks/models/qwen3p7-plus", + "Qwen3.7-Plus (Fireworks, FC)", "Qwen", "proprietary", True, False, + ), +} + +added, updated = [], [] +for name, (model_id, display, org, lic, is_fc, us2dot) in ENTRIES.items(): + # Remove any existing block for this name so a corrected field (e.g. + # underscore_to_dot) actually takes effect on re-run. Append-only would + # silently keep the stale value. + existing = re.search( + r'^ "' + re.escape(name) + r'": ModelConfig\(\n(?:.*?\n)*? \),\n', + src, re.M) + if existing: + src = src[:existing.start()] + src[existing.end():] + updated.append(name) + block = ( + f' "{name}": ModelConfig(\n' + f' model_name="{model_id}",\n' + f' display_name="{display}",\n' + f' url="{("https://openrouter.ai/" + model_id) if name.startswith("openrouter-") else ("https://fireworks.ai/models/fireworks/" + model_id.rsplit("/", 1)[-1])}",\n' + f' org="{org}",\n' + f' license="{lic}",\n' + f' model_handler={"OpenRouterHandler" if name.startswith("openrouter-") else "FireworksHandler"},\n' + f' input_price=None,\n' + f' output_price=None,\n' + f' is_fc_model={is_fc},\n' + f' underscore_to_dot={us2dot},\n' + f' ),\n' + ) + # Append just before the closing brace of api_inference_model_map. + m = re.search(r"^api_inference_model_map = \{$", src, re.M) + if not m: + sys.exit("could not find api_inference_model_map") + close = src.index("\n}\n", m.end()) + src = src[:close + 1] + block + src[close + 1:] + added.append(name) + +if added: + path.write_text(src) + fresh = [a for a in added if a not in updated] + if fresh: + print("[register] added: " + ", ".join(fresh)) + if updated: + print("[register] refreshed: " + ", ".join(updated)) +PY + +echo +echo "Setup complete. Registered models:" +"$VENV/bin/python" - <<'PY' +from bfcl_eval.constants.model_config import MODEL_CONFIG_MAPPING +for k in sorted(MODEL_CONFIG_MAPPING): + if k.startswith(("fireworks-", "openrouter-")): + c = MODEL_CONFIG_MAPPING[k] + print(f" {k:34} -> {c.model_name:48} is_fc={c.is_fc_model}") +PY diff --git a/scripts/sheet_columns.py b/scripts/sheet_columns.py new file mode 100644 index 0000000..d28665e --- /dev/null +++ b/scripts/sheet_columns.py @@ -0,0 +1,209 @@ +""" +Emit the Inkling / Inkling-Small columns for the "running" tab of the +Interfaze Benchmarks sheet, read straight from results/*_metrics.json. + + uv run python scripts/sheet_columns.py + +Writes results/sheet_inkling_columns.tsv and prints the same thing, in the +sheet's row order, so the two columns can be pasted in as a block. + +A cell is left BLANK where our run is not comparable to that row's stated +configuration — a blank is recoverable, a mismatched number silently corrupts +the leaderboard. Every blank is explained in the NOTES column. +""" + +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +RESULTS = ROOT / "results" +OUT = RESULTS / "sheet_inkling_columns.tsv" + + +def j(name): + p = RESULTS / name + return json.loads(p.read_text()) if p.exists() else None + + +def main(): + ocr = j("ocrbench_v2_fireworks_inkling_metrics.json") + mmmu_std = j( + "mmmupro_standard_fireworks_accounts-fireworks-models-inkling_reasoningoff_metrics.json" + ) + mmmu_vis = j( + "mmmupro_vision_fireworks_accounts-fireworks-models-inkling_reasoningoff_metrics.json" + ) + mmmlu = j( + "mmmlulite_fireworks_accounts-fireworks-models-inkling_reasoningoff_metrics.json" + ) + gpqa = j("fireworks_inkling_thinkingoff_gpqa_diamond_metrics.json") + spider = j("spider2_lite_local_fireworks_inkling_metrics.json") + asr = j("voxpopuli_aa_fireworks_accounts_fireworks_models_inkling_metrics.json") + + mmmu_composite = (mmmu_std["accuracy"] + mmmu_vis["accuracy"]) / 2 * 100 + + # (sheet Benchmark label, Inkling cell, Inkling Small cell, note) + rows = [ + ( + "OCRBench V2", + f"{ocr['en_overall'] * 100:.2f}", + "", + "Inkling = EN overall, n=10000. Sheet row says Thinking OFF; Inkling cannot " + "disable thinking, this is reasoning_effort=none (its floor). Small: not measured.", + ), + ( + "olmOCR", + "74.90", + "", + "1403/1403 pages, 8413 tests, +/-1.1. Small: not measured.", + ), + ( + "RefCoco", + "", + "", + "BLANK ON PURPOSE: this row is TestA; our run is val " + f"(strict {32.37:.2f}, format-tolerant oracle {81.19:.2f}). Not the same split — " + "say the word and I'll run TestA to fill it properly.", + ), + ( + "VoxPoppuliCleaned-AA", + f"{(1 - asr['corpus_wer']) * 100:.2f}", + "", + "Reported as 1-WER to match this row's scale (corpus WER 4.61%, n=628). " + "Small: not measured.", + ), + ( + "TheSOB", + "", + "", + "BLANK: TheSOB is not implemented in this repo, so neither model was run.", + ), + ( + "Spider-2.0-lite", + f"{spider['accuracy_of_local_135'] * 100:.2f}", + "", + "64/135 SQLite subset. 5 of the 71 misses hit a 120s per-query cap added to the " + "scorer. Small: not measured.", + ), + ( + "GPQA-Diamond", + f"{gpqa['accuracy'] * 100:.2f}", + "88.30", + "CONFIG MISMATCH: this row is Thinking On; our Inkling run is the floor " + "(mean 6537 reasoning tokens/sample even so). Small 88.30 is Thinking Machines' " + "PUBLISHED figure at effort=0.99, NOT our measurement.", + ), + ( + "MMMLU", + f"{mmmlu['macro_accuracy'] * 100:.2f}", + "", + "CONFIG MISMATCH: row is Thinking On; ours is the floor. Macro over 14 langs, " + "n=19950. Small: not measured.", + ), + ( + "MMMU-Pro", + f"{mmmu_composite:.2f}", + "73.10", + f"Inkling = standard+vision composite ({mmmu_std['accuracy'] * 100:.2f} / " + f"{mmmu_vis['accuracy'] * 100:.2f}), the published convention. Small 73.10 is the " + "PUBLISHED standard-setting figure, not measured, and not a composite.", + ), + ] + + lines = ["Benchmark\tInkling\tInkling Small\tNOTES"] + lines += ["\t".join(r) for r in rows] + OUT.write_text("\n".join(lines) + "\n") + + w = max(len(r[0]) for r in rows) + print(f"{'Benchmark'.ljust(w)} {'Inkling':>8} {'Inkling Small':>13}") + print("-" * (w + 27)) + for label, ink, small, _ in rows: + print(f"{label.ljust(w)} {ink or '(blank)':>8} {small or '(blank)':>13}") + print(f"\nwrote {OUT}") + + + + +# --------------------------------------------------------------------------- +# Extra tabs. The "running" tab above is one column per model; these tabs are +# one ROW per model, with sub-scores as columns. Emitted as separate TSV blocks +# so each can be pasted as a row into its own tab. +# --------------------------------------------------------------------------- + +OUT_TABS = RESULTS / "sheet_inkling_tabs.tsv" + + +def emit_tabs(): + ocr = j("ocrbench_v2_fireworks_inkling_metrics.json") + gpqa = j("fireworks_inkling_thinkingoff_gpqa_diamond_metrics.json") + mmmlu = j("mmmlulite_fireworks_accounts-fireworks-models-inkling_reasoningoff_metrics.json") + spider = j("spider2_lite_local_fireworks_inkling_metrics.json") + en = ocr["en_scores"] + + def p(x): + return f"{x * 100:.2f}" + + blocks = [] + + # --- OCRBench V2 tab: model | Avg Real | Recog | Refer | Spot | Extract + # | Parse | Calc | Understand | Reason (EN scores; matches the tab's + # Interfaze row, whose Avg Real equals the running-tab OCRBench value) + blocks.append(("OCRBench V2 tab", + ["model", "Avg Real", "Recog", "Refer", "Spot", "Extract", "Parse", + "Calc", "Understand", "Reason"], + ["Inkling", p(ocr["en_overall"]), + p(en["text_recognition"]["avg"]), p(en["text_detection"]["avg"]), + p(en["text_spotting"]["avg"]), p(en["relationship_extraction"]["avg"]), + p(en["element_parsing"]["avg"]), p(en["mathematical_calculation"]["avg"]), + p(en["visual_text_understanding"]["avg"]), p(en["knowledge_reasoning"]["avg"])])) + + # --- olmOCR tab. olmOCR-bench prints its per-split table to stdout instead + # of persisting a metrics file, so these come from the run log + # (logs/fireworks_inkling/olmocr.log, 1403/1403 pages, 8413 tests). + blocks.append(("olmOCR tab", + ["model", "Overall", "ArXiv", "OldScansMath", "Tables", "OldScans", + "Headers", "MultiCol", "LongTinyText", "Base", "Real Overall"], + ["Inkling", "74.90", "71.20", "78.80", "82.60", "41.60", "91.60", + "72.40", "61.80", "99.10", "74.9±1.1"])) + + # --- GPQA tab: model | Overall | Physics (n=86) | Chemistry (n=93) | Biology (n=19) + d = gpqa["per_domain"] + blocks.append(("GPQA tab", + ["model", "Overall", "Physics (n=86)", "Chemistry (n=93)", "Biology (n=19)"], + ["Inkling", p(gpqa["accuracy"]), p(d["Physics"]["accuracy"]), + p(d["Chemistry"]["accuracy"]), p(d["Biology"]["accuracy"])])) + + # --- MMMLU tab: languages are ROWS and models are COLUMNS, so this is a + # column (macro first, then the 14 languages in the sheet's order). + order = ["FR_FR", "PT_BR", "BN_BD", "JA_JP", "DE_DE", "YO_NG", "ES_LA", + "ID_ID", "ZH_CN", "SW_KE", "IT_IT", "AR_XY", "KO_KR", "HI_IN"] + pl = mmmlu["per_language"] + blocks.append(("MMMLU tab (column: macro then langs)", + ["lang", "Inkling"], + None)) + mmmlu_rows = [["macro", p(mmmlu["macro_accuracy"])]] + mmmlu_rows += [[lg, p(pl[lg]["accuracy"])] for lg in order if lg in pl] + + # --- Spider-2.0 tab: Model | Spider-2.0-lite + blocks.append(("Spider-2.0 tab", + ["Model", "Spider-2.0-lite"], + ["Inkling", p(spider["accuracy_of_local_135"])])) + + lines = [] + for title, header, row in blocks: + lines.append(f"# {title}") + lines.append("\t".join(header)) + if row: + lines.append("\t".join(row)) + else: + lines += ["\t".join(r) for r in mmmlu_rows] + lines.append("") + OUT_TABS.write_text("\n".join(lines) + "\n") + print("\n".join(lines)) + print(f"wrote {OUT_TABS}") + + +if __name__ == "__main__": + main() + print() + emit_tabs() diff --git a/src/commons.py b/src/commons.py index 284e47b..e18cabc 100644 --- a/src/commons.py +++ b/src/commons.py @@ -7,15 +7,17 @@ load_dotenv() -if (INTERFAZE_API_KEY := os.getenv("INTERFAZE_API_KEY", None)) is None: - raise ValueError( - "INTERFAZE_API_KEY is not set in environment variables get it from https://interfaze.ai/dashboard" - ) +INTERFAZE_API_KEY = os.getenv("INTERFAZE_API_KEY", None) INTERFAZE_BASE_URL = os.getenv("OPENAI_BASE_URL", "https://api.interfaze.ai/v1") -interfaze_client = OpenAI( - base_url=INTERFAZE_BASE_URL, api_key=os.getenv("INTERFAZE_API_KEY") +# Built only when the key is present. The multi-provider runners import shared +# helpers from the interfaze scripts, so a hard raise here would block running +# any other provider; the error surfaces on first use instead. +interfaze_client = ( + OpenAI(base_url=INTERFAZE_BASE_URL, api_key=INTERFAZE_API_KEY) + if INTERFAZE_API_KEY + else None ) @@ -25,6 +27,8 @@ def invoke_interfaze( stream: bool = False, structured_response: bool = False, structure_definition: BaseModel | None = None, + reasoning_effort: str | None = None, + temperature: float | None = None, ) -> dict: """For invoking interfaze with images you can use messages = [ @@ -42,13 +46,16 @@ class ResponseModel(BaseModel): structure_definition=ResponseModel ) """ - try: - response = interfaze_client.chat.completions.create( - model=model, - messages=messages, - stream=stream, + if interfaze_client is None: + raise ValueError( + "INTERFAZE_API_KEY is not set in environment variables get it from https://interfaze.ai/dashboard" ) - - return response + kwargs: dict = {"model": model, "messages": messages, "stream": stream} + if reasoning_effort is not None: + kwargs["reasoning_effort"] = reasoning_effort + if temperature is not None: + kwargs["temperature"] = temperature + try: + return interfaze_client.chat.completions.create(**kwargs) except Exception as e: raise RuntimeError(f"Error invoking Interfaze API: {e}") from e diff --git a/src/commons_fireworks.py b/src/commons_fireworks.py new file mode 100644 index 0000000..e62f0d1 --- /dev/null +++ b/src/commons_fireworks.py @@ -0,0 +1,66 @@ +import os + +from openai import OpenAI +from dotenv import load_dotenv + +load_dotenv() + +# The project .env may spell this key lowercase (`fireworks_api_key`); +# load_dotenv() copies the name verbatim, so accept either casing. +FIREWORKS_API_KEY = os.getenv("FIREWORKS_API_KEY") or os.getenv("fireworks_api_key") + +if FIREWORKS_API_KEY is None: + raise ValueError( + "FIREWORKS_API_KEY is not set in environment variables get it from https://app.fireworks.ai/settings/users/api-keys" + ) + +FIREWORKS_BASE_URL = os.getenv( + "FIREWORKS_BASE_URL", "https://api.fireworks.ai/inference/v1" +) + +# Inkling (Thinking Machines) — text+image+audio in, text out, 1M context. +# inkling-small is NOT on Fireworks serverless; it needs an on-demand +# dedicated deployment, after which its model id is the same string. +INKLING = "accounts/fireworks/models/inkling" +INKLING_SMALL = "accounts/fireworks/models/inkling-small" + +# Inkling always thinks. reasoning_effort accepts none|low|medium|high|xhigh|max +# — "none" is the floor (still emits reasoning tokens), there is no true off. +REASONING_FLOOR = "none" +REASONING_MAX = "high" + +fireworks_client = OpenAI(api_key=FIREWORKS_API_KEY, base_url=FIREWORKS_BASE_URL) + + +def invoke_fireworks( + messages: list[dict], + model: str = INKLING, + reasoning_effort: str | None = None, + temperature: float | None = None, + max_tokens: int | None = None, + stream: bool = False, +): + """Invoke a Fireworks-hosted model over the OpenAI-compatible endpoint. + + Image input uses the standard `image_url` block with a data URL. Audio + input uses an `audio_url` block with a data URL (Fireworks rejects + OpenAI's `input_audio` shape for this model): + messages = [{ + "role": "user", + "content": [ + {"type": "text", "text": "Transcribe this audio."}, + {"type": "audio_url", "audio_url": {"url": "data:audio/wav;base64,..."}}, + ], + }] + """ + kwargs: dict = {"model": model, "messages": messages, "stream": stream} + if reasoning_effort is not None: + kwargs["reasoning_effort"] = reasoning_effort + if temperature is not None: + kwargs["temperature"] = temperature + if max_tokens is not None: + kwargs["max_tokens"] = max_tokens + try: + return fireworks_client.chat.completions.create(**kwargs) + except Exception as e: + raise RuntimeError(f"Error invoking Fireworks API: {e}") from e diff --git a/src/commons_openrouter.py b/src/commons_openrouter.py index 934641e..4d0452d 100644 --- a/src/commons_openrouter.py +++ b/src/commons_openrouter.py @@ -6,14 +6,26 @@ load_dotenv() -if (OPENROUTER_API_KEY := os.getenv("OPENROUTER_API_KEY", None)) is None: +# This project's .env spells the key lowercase; accept either casing. +if ( + OPENROUTER_API_KEY := os.getenv("OPENROUTER_API_KEY") + or os.getenv("openrouter_api_key") +) is None: raise ValueError( "OPENROUTER_API_KEY is not set in environment variables get it from https://openrouter.ai/keys" ) OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1" -openrouter_client = OpenAI(api_key=OPENROUTER_API_KEY, base_url=OPENROUTER_BASE_URL) +# Explicit timeout + retries: without them a single hung connection blocks +# forever. Observed on Spider2 — one request never returned and stalled the +# whole benchmark queue for 5h at 0% CPU with 134/135 already done. +openrouter_client = OpenAI( + api_key=OPENROUTER_API_KEY, + base_url=OPENROUTER_BASE_URL, + timeout=180.0, + max_retries=2, +) def invoke_openrouter( diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..5767660 --- /dev/null +++ b/uv.lock @@ -0,0 +1,4132 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", + "python_full_version < '3.13' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] + +[[package]] +name = "aiohappyeyeballs" +version = "2.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/f4/eec0465c2f67b2664688d0240b3212d5196fd89e741df67ddb81f8d35658/aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d", size = 24757, upload-time = "2026-07-01T17:11:55.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", size = 15038, upload-time = "2026-07-01T17:11:54.055Z" }, +] + +[[package]] +name = "aiohttp" +version = "3.14.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs" }, + { name = "aiosignal" }, + { name = "attrs" }, + { name = "frozenlist" }, + { name = "multidict" }, + { name = "propcache" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/d9/22ce5786ac0c1653ae8b6c23bded02c1686d11f0dbb45b31ce128e0df985/aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc", size = 7971213, upload-time = "2026-07-23T01:57:27.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/d4/eb96299230e20acf2efae207cb8d69051f1f68e357e5ea5e479bf6fb097a/aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5", size = 754690, upload-time = "2026-07-23T01:53:47.332Z" }, + { url = "https://files.pythonhosted.org/packages/88/11/e7a70a209eb9a067c0d3212b518a0134e3484f5178c7533878b6b514d469/aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228", size = 509484, upload-time = "2026-07-23T01:53:51.159Z" }, + { url = "https://files.pythonhosted.org/packages/30/07/4bbc222cc8dbe31d4c3e8a5baad2286e4d42026ac0c570027b89afce6344/aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee", size = 511949, upload-time = "2026-07-23T01:53:55.083Z" }, + { url = "https://files.pythonhosted.org/packages/54/b9/42e74c46b7b7c794b995bbc1f573fb48950c38b19d8600c62a6804ee2d67/aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a", size = 1765282, upload-time = "2026-07-23T01:53:59.662Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ed/62bc4d74363ad346d518e0720363a949f63e2e23439a79eb5813d4d29bb3/aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b", size = 1741511, upload-time = "2026-07-23T01:54:04.063Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9f/181e8a8bc79e47d13c7fc4540bd7a3b729d9505609c61f392a8dd2fbfe55/aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529", size = 1810680, upload-time = "2026-07-23T01:54:09.882Z" }, + { url = "https://files.pythonhosted.org/packages/5c/9a/dec94d6ad694552fe3424e3f1928d7a606a5d9d9433a04e7ecdd9d38ae7f/aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787", size = 1905646, upload-time = "2026-07-23T01:54:13.475Z" }, + { url = "https://files.pythonhosted.org/packages/52/b7/7cd31f29d6055bd711ae6e669367fba6f5ae9de463910a793e30556a8db7/aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42", size = 1792122, upload-time = "2026-07-23T01:54:15.752Z" }, + { url = "https://files.pythonhosted.org/packages/66/73/10b1ef93afa61f4963c746257b70ced619cf31a4798671de5fdb2608501d/aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b", size = 1591127, upload-time = "2026-07-23T01:54:19.489Z" }, + { url = "https://files.pythonhosted.org/packages/49/ed/3b203fa6de1b338c14acdc06bf6ca9b043b7944f005966958c2ced932cde/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043", size = 1725210, upload-time = "2026-07-23T01:54:24.129Z" }, + { url = "https://files.pythonhosted.org/packages/28/b7/1c2aab8c706436dcc28598452488ac9cd7c409da815237c28c27d58993e6/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427", size = 1764848, upload-time = "2026-07-23T01:54:27.973Z" }, + { url = "https://files.pythonhosted.org/packages/54/50/94c28f08b131c4bf10984ea2c7a536c9920608bb2d6e7f95642c30cc87b7/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d", size = 1777102, upload-time = "2026-07-23T01:54:31.775Z" }, + { url = "https://files.pythonhosted.org/packages/13/d4/e7d09ba7d345fb2d74440fd2fa033c5e079fac05552927705986f41a364f/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0", size = 1580205, upload-time = "2026-07-23T01:54:34.518Z" }, + { url = "https://files.pythonhosted.org/packages/a3/84/072a91d68e1e1eb587985b54baab94221277f877e8ef274fc213a0ceae28/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d", size = 1797219, upload-time = "2026-07-23T01:54:36.995Z" }, + { url = "https://files.pythonhosted.org/packages/e0/eb/aad34e897e668424d6e995da5dff8a4a09af93363d3392488772957a63aa/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19", size = 1768629, upload-time = "2026-07-23T01:54:40.103Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2b/6bb88ddba0fecd9122aa3ebcad25996cf6c083a4a7040dbb3a4f97972af6/aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559", size = 451481, upload-time = "2026-07-23T01:54:42.547Z" }, + { url = "https://files.pythonhosted.org/packages/76/9b/f2f8f108da17ecef2cc3efc424e8b7ad3782b1a8360f7b8eae8ced84f6ea/aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a", size = 476845, upload-time = "2026-07-23T01:54:44.853Z" }, + { url = "https://files.pythonhosted.org/packages/3e/44/28dac80a8941b604f4da10ce21097614ca1bf905ce93dca28d8d7de9c1e7/aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c", size = 448050, upload-time = "2026-07-23T01:54:47.087Z" }, + { url = "https://files.pythonhosted.org/packages/57/be/5afd201cc0ab139029aadb75392efe85a293403d9dd3a3226161c21ce00c/aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86", size = 506269, upload-time = "2026-07-23T01:54:49.075Z" }, + { url = "https://files.pythonhosted.org/packages/22/09/dec8189d62b45ade009f6792a2264b942a90cb88aeaf181239933cd72c3c/aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627", size = 515166, upload-time = "2026-07-23T01:54:51.894Z" }, + { url = "https://files.pythonhosted.org/packages/28/24/2854869d29ed8a8b19d74f9ec6629515f7e04d02dd329d9d179201e58e47/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82", size = 486263, upload-time = "2026-07-23T01:54:54.223Z" }, + { url = "https://files.pythonhosted.org/packages/d4/dd/57187c8be2a35aea65eaee3bd2c3dcbbcf0204f5106c89637e3610380cd1/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c", size = 492299, upload-time = "2026-07-23T01:54:56.236Z" }, + { url = "https://files.pythonhosted.org/packages/b9/11/06ae6ed8f0d414edf4068861e233d8fe23ee699bfd4b3ceb8663db948a62/aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f", size = 502235, upload-time = "2026-07-23T01:54:58.377Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a3/559639c34a345d2cf7c52dff6838119f2eaf29eb508227b5b83f573af813/aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80", size = 750883, upload-time = "2026-07-23T01:55:00.65Z" }, + { url = "https://files.pythonhosted.org/packages/91/cd/41e131f13afd1e7b0172a9d9eda085ef90eb8439f41f0d279db81ed3ae60/aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0", size = 508473, upload-time = "2026-07-23T01:55:02.945Z" }, + { url = "https://files.pythonhosted.org/packages/bc/6b/e7f13410d391c6e55b4c007a8de024355389d7d459e3d64c42b2d33617e5/aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf", size = 509190, upload-time = "2026-07-23T01:55:05.173Z" }, + { url = "https://files.pythonhosted.org/packages/97/21/6464573e53d69672cc1eada3e5c5cb2d2efa82701e8305a0f2047a576967/aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd", size = 1761478, upload-time = "2026-07-23T01:55:07.383Z" }, + { url = "https://files.pythonhosted.org/packages/1a/81/d217043a4c17fbce360905e3b2bdd20139ebc9a2de836d035d179c4da006/aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807", size = 1735092, upload-time = "2026-07-23T01:55:09.803Z" }, + { url = "https://files.pythonhosted.org/packages/a1/66/e13a02d0eeb1a9a502402a977abb4e4abff9fe4051c26f80558c57a7c975/aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8", size = 1800546, upload-time = "2026-07-23T01:55:12.012Z" }, + { url = "https://files.pythonhosted.org/packages/26/5e/57d42fca1d18cb5acc1cad945d017fabc5d6ae71d8a08ad66be8dc3ee544/aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24", size = 1895250, upload-time = "2026-07-23T01:55:14.357Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1c/7da8d08e74d56f00070822f9638ff3f1c563f8ad87d1efa996c87bfc8644/aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5", size = 1789289, upload-time = "2026-07-23T01:55:16.668Z" }, + { url = "https://files.pythonhosted.org/packages/cd/0f/cf16bcf56896981c1a0319f5d5db9337994b5165730c48a8fa07e9b34be6/aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4", size = 1586706, upload-time = "2026-07-23T01:55:18.913Z" }, + { url = "https://files.pythonhosted.org/packages/fe/6f/76eac12a7f2480e1e304f842efdb07db33256b0d9165b866b6ef0806c202/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9", size = 1724652, upload-time = "2026-07-23T01:55:21.296Z" }, + { url = "https://files.pythonhosted.org/packages/39/b6/19c8c592baeeb94b75f966547d40c02ac7590902306ec5863d5c027cf506/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1", size = 1756239, upload-time = "2026-07-23T01:55:23.705Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c9/4e9383150296f97f873b680c4de8fb2cd88608fb9f48c79edcb111611abc/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371", size = 1769161, upload-time = "2026-07-23T01:55:26.082Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1e/147bdc6cc5de5f3ab011be8bf5d6e786633249f22c20bae06f85e45f5387/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde", size = 1578759, upload-time = "2026-07-23T01:55:28.846Z" }, + { url = "https://files.pythonhosted.org/packages/fd/31/78388a9d6040ece2e11df62ea229a822cf5e52d238374b220ae9975b2623/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e", size = 1792025, upload-time = "2026-07-23T01:55:31.457Z" }, + { url = "https://files.pythonhosted.org/packages/03/51/a3d29fdf2c25d796746af8ad6fe56a45d6256c38b0a8a2ed752e1160b3a2/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71", size = 1768477, upload-time = "2026-07-23T01:55:33.87Z" }, + { url = "https://files.pythonhosted.org/packages/29/a6/442e18b5afeade534d877a2dc3c3e392aff8d49787890b0cf84790410267/aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0", size = 451069, upload-time = "2026-07-23T01:55:36.121Z" }, + { url = "https://files.pythonhosted.org/packages/9d/69/3d876ac02659f271cf7f6769f14a8e3de5b6e888ed8b5a7e998086a4cec8/aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883", size = 476518, upload-time = "2026-07-23T01:55:38.303Z" }, + { url = "https://files.pythonhosted.org/packages/b2/0e/50d6e6471cd31edce8b282bdec59375a3a69124d8a989a0b1313355cae52/aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2", size = 447676, upload-time = "2026-07-23T01:55:40.451Z" }, + { url = "https://files.pythonhosted.org/packages/c8/20/887fdcf832326571b370ffc347b3e70abe101096f3720126aac161b1d872/aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062", size = 509067, upload-time = "2026-07-23T01:55:42.618Z" }, + { url = "https://files.pythonhosted.org/packages/ad/a3/92cec936f78cc4bf0fa5554ebe593b73459d94e3c62303e1902a4cccb6f7/aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6", size = 514774, upload-time = "2026-07-23T01:55:44.937Z" }, + { url = "https://files.pythonhosted.org/packages/29/ba/2a0c38df3fc557620b6a5acd98364af050053b6285b4dc7ee74100c63c18/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919", size = 488134, upload-time = "2026-07-23T01:55:47.135Z" }, + { url = "https://files.pythonhosted.org/packages/48/d6/d51b7d4bf309af3693940d8ffd2b9ed0b682434ef85959b7c9c137f60cf8/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7", size = 494201, upload-time = "2026-07-23T01:55:49.451Z" }, + { url = "https://files.pythonhosted.org/packages/3f/5a/8f624384e5f1efabb5229b94157eb966b021e97bdb188c62860c2ae243c2/aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0", size = 502766, upload-time = "2026-07-23T01:55:51.656Z" }, + { url = "https://files.pythonhosted.org/packages/a6/26/4ff0164370deec18fb19254ee4ab10b7a73304ac0c860b13f5f84663759b/aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924", size = 756557, upload-time = "2026-07-23T01:55:53.964Z" }, + { url = "https://files.pythonhosted.org/packages/97/a3/7056b86dc0d9ec709ea9777eae3b0161428f943372f8b98c01c11593b682/aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646", size = 510168, upload-time = "2026-07-23T01:55:56.22Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/0357a015892fd68058bf2d39d3fd1958e459b997a7db30aaa6aaa434ae96/aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b", size = 512957, upload-time = "2026-07-23T01:55:58.437Z" }, + { url = "https://files.pythonhosted.org/packages/47/d1/8aba53f15ccb2238405f5e9d30e2a8ca44f93878c26e7165ade00d374b1c/aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30", size = 1750149, upload-time = "2026-07-23T01:56:00.856Z" }, + { url = "https://files.pythonhosted.org/packages/49/bd/40c3fee327529284375c6701cbb0fa4600cc2e8432af1378f897e2ef7d3a/aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9", size = 1707685, upload-time = "2026-07-23T01:56:03.371Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a3/ca0cc6724cca8114b05694abd916060758c79894c3aa5b012cdadc1bc28e/aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f", size = 1803911, upload-time = "2026-07-23T01:56:05.817Z" }, + { url = "https://files.pythonhosted.org/packages/95/b5/85b099c299c3ffd38ad9b3e43694c8a346934e4a30c88c4fd5a841234f77/aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d", size = 1876929, upload-time = "2026-07-23T01:56:08.413Z" }, + { url = "https://files.pythonhosted.org/packages/d5/b7/1da684a04175473fa4cddbf9a2f572e79514c3fd27a74597f43057d4f3da/aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147", size = 1761112, upload-time = "2026-07-23T01:56:10.918Z" }, + { url = "https://files.pythonhosted.org/packages/d1/16/bc4b55e3e5cb175fd69c53c90d60d2f47797cb343da5106e23863dc4dba4/aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c", size = 1583500, upload-time = "2026-07-23T01:56:13.613Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e8/13a9d957a1ee40837f46aa30f0f4c657e673ad86a2e6362a9f9be20d26d9/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a", size = 1713940, upload-time = "2026-07-23T01:56:15.969Z" }, + { url = "https://files.pythonhosted.org/packages/38/05/d33c680c1bcf1c7e130f9cbfc1fc02fe8bb0c4af2a94a53dd5fb56131e5c/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0", size = 1724413, upload-time = "2026-07-23T01:56:18.591Z" }, + { url = "https://files.pythonhosted.org/packages/85/1d/af798d306f7a74b6a632dbcabcf62a4c91391b7582d2a8c6d7712e2cc54e/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661", size = 1770748, upload-time = "2026-07-23T01:56:21.074Z" }, + { url = "https://files.pythonhosted.org/packages/a8/92/ad720d472556a995049206867765e9410969684f86ee09423ff9969044c1/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22", size = 1577564, upload-time = "2026-07-23T01:56:23.475Z" }, + { url = "https://files.pythonhosted.org/packages/60/ad/0ed7586cbef7a884e23a752fa2bb987a122e6a5dd50dab109258d0a95193/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41", size = 1782080, upload-time = "2026-07-23T01:56:25.994Z" }, + { url = "https://files.pythonhosted.org/packages/97/ea/dbaed0d73e8a69aad653b045dab451c67c2454bb731a37b45a86593e9422/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf", size = 1745813, upload-time = "2026-07-23T01:56:28.604Z" }, + { url = "https://files.pythonhosted.org/packages/81/1b/6893d4bc57e434fc93a6c9217c637d967a0b651d989f6e3265179375754a/aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da", size = 455872, upload-time = "2026-07-23T01:56:31.031Z" }, + { url = "https://files.pythonhosted.org/packages/f5/8b/c7baa1ba1eda4db6989baefe5de6d99834921b84ebd7918624febcb9f290/aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100", size = 481030, upload-time = "2026-07-23T01:56:33.365Z" }, + { url = "https://files.pythonhosted.org/packages/22/8c/c29d067df825a2df88ca432db848aa2fe8199598359cc06c12b09320cac9/aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc", size = 453669, upload-time = "2026-07-23T01:56:35.731Z" }, + { url = "https://files.pythonhosted.org/packages/6a/a4/9c033beb355d39b6147980597ec9645e4729243f686ee4dc73945de72030/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b", size = 791403, upload-time = "2026-07-23T01:56:37.972Z" }, + { url = "https://files.pythonhosted.org/packages/80/ca/87c32a0a7704583cfc49660bd817889bae5b830bf53b5dcb4e92145ac2da/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0", size = 526413, upload-time = "2026-07-23T01:56:40.523Z" }, + { url = "https://files.pythonhosted.org/packages/9e/d8/8ec0e471248c500acdce2be3f46db8fb62b5eb60efef072529cc85ee1d26/aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e", size = 532135, upload-time = "2026-07-23T01:56:42.876Z" }, + { url = "https://files.pythonhosted.org/packages/fe/45/f8919fd936e8b79fcd9bda7b6d8e62613462a713f4f17987fd7c34399142/aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716", size = 1922742, upload-time = "2026-07-23T01:56:45.528Z" }, + { url = "https://files.pythonhosted.org/packages/f6/ec/9ca76b28a27525b0cc53e20842e0228b022f301ce1f436b7d814b4aaf2df/aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f", size = 1787371, upload-time = "2026-07-23T01:56:48.045Z" }, + { url = "https://files.pythonhosted.org/packages/b1/04/6acdbf17315f7b55f1937e3387acb89a3cddeb4995689553d064af8e92ab/aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553", size = 1912623, upload-time = "2026-07-23T01:56:50.605Z" }, + { url = "https://files.pythonhosted.org/packages/86/e6/438b0c79ca6f45eb9fd9817dd4c01a91919a38c0de5ee9e05e2b4dc0ece7/aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100", size = 2005515, upload-time = "2026-07-23T01:56:53.153Z" }, + { url = "https://files.pythonhosted.org/packages/bb/6b/62cbd6577758699525f5c712d1ddef57d9875fbab0ae8d5f5a202fd598f8/aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85", size = 1879906, upload-time = "2026-07-23T01:56:55.818Z" }, + { url = "https://files.pythonhosted.org/packages/00/95/18bcbf830a21dc3aae24d8f6b6feaf3db1d2090242d00a7868db2ffb0b67/aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33", size = 1675849, upload-time = "2026-07-23T01:56:58.861Z" }, + { url = "https://files.pythonhosted.org/packages/a9/19/47f4968659c5e23606c3790c80fc624e691c153d036148449ee84d31b287/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f", size = 1843496, upload-time = "2026-07-23T01:57:01.591Z" }, + { url = "https://files.pythonhosted.org/packages/64/af/38c33c4dd82fddcb4e56c4653b6f1072a8edbc6b7fa15809f14932c41e2d/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0", size = 1827746, upload-time = "2026-07-23T01:57:05.131Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9d/0537cda4885ac8f5b7053d164dd06312f4c483a4edcb8ee5b8aaf2a989bf/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098", size = 1853810, upload-time = "2026-07-23T01:57:08.043Z" }, + { url = "https://files.pythonhosted.org/packages/19/fe/26f9c5e6458385aa86497836b0dea6fb2f027827d63f37c7856cce9286ee/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25", size = 1668895, upload-time = "2026-07-23T01:57:10.837Z" }, + { url = "https://files.pythonhosted.org/packages/ec/4c/618b1db9b9ba079b8875d2cdf78e7c4a3bf72903bd5850fee7dd9544600a/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9", size = 1883833, upload-time = "2026-07-23T01:57:13.672Z" }, + { url = "https://files.pythonhosted.org/packages/94/c6/bd959bd1e4771f9fd944e9e436224c48c77b018b73b519b5aad346335bcc/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb", size = 1844251, upload-time = "2026-07-23T01:57:16.593Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/08d41839658bdd44a0ed2480f3891705ecb487ce28c0dde62c9040c997e0/aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963", size = 474180, upload-time = "2026-07-23T01:57:19.306Z" }, + { url = "https://files.pythonhosted.org/packages/99/5d/3cd6ef0a2b2851f7ab913b5b079334781bd50ff56a323e4454063377a080/aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b", size = 500528, upload-time = "2026-07-23T01:57:21.762Z" }, + { url = "https://files.pythonhosted.org/packages/a4/37/cfd1ed540a4d318da025590d96b728e63713c09e9377950fc655dadeb856/aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7", size = 469280, upload-time = "2026-07-23T01:57:24.241Z" }, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "anyio" +version = "4.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, +] + +[[package]] +name = "apted" +version = "1.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/29/3a42b2fb26272a464a9fbf455928a7e4255efa2e6f56679e9c0adaaf798a/apted-1.0.3.tar.gz", hash = "sha256:befa5181e2d4457fa88e54995a82604ee048bb2fbc781ea97d8e1856b4715ce9", size = 24547, upload-time = "2017-11-08T13:03:23.294Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/71/c2bcf92376d3ae65d57111d33f577aca68d343e1b7b1914a3767bfbac18e/apted-1.0.3-py3-none-any.whl", hash = "sha256:74193369d023649d335269e67c4df07f922959e5ac2597de1b79af4e694150e8", size = 40566, upload-time = "2017-11-08T13:03:21.831Z" }, +] + +[[package]] +name = "asttokens" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/25/1e/faf0f247f6f881b98fc4d6d07e14085cb89d13665084e6d6ac1dc2c03d0b/asttokens-3.0.2.tar.gz", hash = "sha256:3ecdbd8f2cc195f53ccada3a613538bb5f9ef6f6869129f13e03c30a677b8fe2", size = 63136, upload-time = "2026-07-12T03:31:49.084Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/2b/04b8a15f3a1c77bc79ddf5c73875327f34b4fa75982df2b76e45e402d364/asttokens-3.0.2-py3-none-any.whl", hash = "sha256:9da13157f5b28becde0bd374fc677dcd3c290614264eff096f167c469cd9f933", size = 28702, upload-time = "2026-07-12T03:31:47.542Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "beautifulsoup4" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, +] + +[[package]] +name = "bleach" +version = "6.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "webencodings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/3c/e12ac860709702bd5ebeb9b56a4fe334f1001246ee1b8f2b7ee28912df7d/bleach-6.4.0.tar.gz", hash = "sha256:4202482733d85cedd04e59fcb2f89f4e4c7c385a78d3c3c23c30446843a37452", size = 204857, upload-time = "2026-06-05T13:01:13.734Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/9d/40b6267367182187139a4000b82a3b287d84d745bccd808e75d916920e9d/bleach-6.4.0-py3-none-any.whl", hash = "sha256:4b6b6a54fff2e69a3dde9d21cc6301220bee3c3cb792187d11403fd795031081", size = 165109, upload-time = "2026-06-05T13:01:12.504Z" }, +] + +[[package]] +name = "boto3" +version = "1.43.63" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4a/b7/3fdd53534170ef7d99d1707071e57ea0aeb0dec84ed0206d31e8c78f54d7/boto3-1.43.63.tar.gz", hash = "sha256:647c0f0b59710ce12a49323382bb5ece1b4e02199c736d19d555330dca7e947f", size = 112658, upload-time = "2026-08-03T19:55:05.178Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/57/a91420a14ef26fe52fc0706b5af21c00b2e4ff1c57b0c4272370c9b80399/boto3-1.43.63-py3-none-any.whl", hash = "sha256:859a8d1c50505a5cefb8629790dd34602ddb85bfd7ea5d34a362ab1793513636", size = 140024, upload-time = "2026-08-03T19:55:03.4Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.63" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/5f/b33913aab846bc88a2720976435adb944d1ef57b92beed829233fe1953d9/botocore-1.43.63.tar.gz", hash = "sha256:854e45247f00b0732496ea1f0c5d0cf3c31d58b48eb052c31c27ab1087dfddf1", size = 15824662, upload-time = "2026-08-03T19:54:55.233Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/ae/8ddbf97a12fba9b6ce50ac1c2209ebd2ab3b240229a1fbe5de66ffcbd05f/botocore-1.43.63-py3-none-any.whl", hash = "sha256:8deed86feacc8f8d2491f1d170562af191f8b33924052d834f4217d5d797e5a9", size = 15509076, upload-time = "2026-08-03T19:54:52.455Z" }, +] + +[[package]] +name = "cached-path" +version = "1.8.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "boto3" }, + { name = "filelock" }, + { name = "google-cloud-storage" }, + { name = "huggingface-hub" }, + { name = "packaging" }, + { name = "requests" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/da/59aa2a3f5d92415d1c072a35c94c7ada2251679b3bca3cf69d420ea95ac4/cached_path-1.8.10.tar.gz", hash = "sha256:ce80db439e25619800330dcbf1f0516c0ee70a27bd65ffca33aac9f55a56ef1c", size = 33249, upload-time = "2026-03-20T17:52:08.713Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/db/ccb08109c7056b0670384fc5042be5eacd1391eb12485188e92ca97ced21/cached_path-1.8.10-py3-none-any.whl", hash = "sha256:a7a80c4a77859e40080ed3450bc1ca5434a74fc51566361677a75fd2f5b8fea8", size = 37932, upload-time = "2026-03-20T17:52:07.747Z" }, +] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, + { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, + { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, + { url = "https://files.pythonhosted.org/packages/9d/f4/035513d4117049066b4779dc3b7c0c0fdad175fa13731c9f4003f1cd1478/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e", size = 194248, upload-time = "2026-08-03T21:19:59.399Z" }, + { url = "https://files.pythonhosted.org/packages/76/af/2aeb4dbb5fc41a04161ae9ff1518de7cec08e164f44a8ce6a4cf7fd2cd1d/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c", size = 196908, upload-time = "2026-08-03T21:20:00.746Z" }, + { url = "https://files.pythonhosted.org/packages/a7/46/2e5fdde8555706dd98139a910ca11be02809f3f605ce956f655d0214e100/cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6", size = 184805, upload-time = "2026-08-03T21:20:02.02Z" }, + { url = "https://files.pythonhosted.org/packages/55/41/4c7042f317b9217502988f0873af87e16ad606dc20f84e546e3e6ce9764c/cffi-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971", size = 184764, upload-time = "2026-08-03T21:20:03.141Z" }, + { url = "https://files.pythonhosted.org/packages/43/1f/1c3d90d91811c8f86ced9ed637956c54bfe5b79ca98fe976d7f8c8979f6b/cffi-2.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c", size = 214722, upload-time = "2026-08-03T21:20:04.377Z" }, + { url = "https://files.pythonhosted.org/packages/37/6f/3b5ce4c3b2192d250f04908f2bfd91ef34552ec8f7716a5d4abdb8d67bb2/cffi-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125", size = 222369, upload-time = "2026-08-03T21:20:05.544Z" }, + { url = "https://files.pythonhosted.org/packages/02/10/4b3c75dde3d9663c9e02ba05c2668b954f671d4bbe346413ca8c696b295a/cffi-2.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264", size = 210175, upload-time = "2026-08-03T21:20:06.75Z" }, + { url = "https://files.pythonhosted.org/packages/df/62/14f74b9543e605d17701dc797b815958b8bb70b7624ce1b832ddad48ed6c/cffi-2.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3", size = 208670, upload-time = "2026-08-03T21:20:08.04Z" }, + { url = "https://files.pythonhosted.org/packages/95/95/86342356ff5953b3fb06f7ef7c5bee212d45e770abc7218d451b9148313c/cffi-2.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2", size = 221824, upload-time = "2026-08-03T21:20:09.274Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ff/7b3429ff53aafe931ed8a5fc69f481bbef7ba6de87ddcbb63d08f483f613/cffi-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b", size = 225148, upload-time = "2026-08-03T21:20:10.7Z" }, + { url = "https://files.pythonhosted.org/packages/34/34/a95870b9221e09cf4f2ce3178b1a210abdfe63a1bd357da940418d7b8d15/cffi-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7", size = 223564, upload-time = "2026-08-03T21:20:12.165Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/839b50531021a647fb5e929f72cf97bc1ff702b5472166164b5b6e76b851/cffi-2.1.1-cp313-cp313-win32.whl", hash = "sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac", size = 175263, upload-time = "2026-08-03T21:20:13.559Z" }, + { url = "https://files.pythonhosted.org/packages/60/a6/8b149b2c3f2e11aaa1618ef64500b45f50f22c57a977a4dff1aff1f91042/cffi-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d", size = 185688, upload-time = "2026-08-03T21:20:14.69Z" }, + { url = "https://files.pythonhosted.org/packages/01/9a/11f687cb39d6a3504060d5242f04f48c735afb4d3d533958a20594890cb2/cffi-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973", size = 180078, upload-time = "2026-08-03T21:20:15.917Z" }, + { url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/d9/99/c4b0c17cacdc9c3b8f280026286a9826d6a208c0f047591a3c3ce99b91fd/cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54", size = 184964, upload-time = "2026-08-03T21:20:19.708Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a9/9db617d05d7367c1ad0ab00b3aa6e6f9281edd689b4ee9ea0e5a84e89c97/cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72", size = 184962, upload-time = "2026-08-03T21:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" }, + { url = "https://files.pythonhosted.org/packages/80/10/c5c0cbf0a657aecf59ef511409734230bf556f05a0d6c9eed7aa5c0a0166/cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062", size = 209985, upload-time = "2026-08-03T21:20:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/d5/6c/bfa0b87b03b9238148beca990292843c9396ba069b54496596594173de7b/cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03", size = 208530, upload-time = "2026-08-03T21:20:24.628Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" }, + { url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" }, + { url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e9/d0061c364cde06ee43168a0d076ac1da512cbc380d44767b844ba34fe2b6/cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c", size = 177682, upload-time = "2026-08-03T21:20:44.288Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1c3e01e3ba14c39f6d10bfbac52753b7e22259e38088e5cfe1d704918690/cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48", size = 187949, upload-time = "2026-08-03T21:20:45.623Z" }, + { url = "https://files.pythonhosted.org/packages/87/5b/da4e39efe18eeb89cf580ea9cfc66b6a7c3eadb808fc0cc1d3a295cb5a5d/cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836", size = 182947, upload-time = "2026-08-03T21:20:46.955Z" }, + { url = "https://files.pythonhosted.org/packages/23/59/40338bf421c5accea1d45158170c87006ef1cd371b05c077e76476949728/cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3", size = 188504, upload-time = "2026-08-03T21:20:29.495Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/5ecf1023850036e674c77ec4de86182d309ae344e39e7cba984b7df5d647/cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2", size = 188259, upload-time = "2026-08-03T21:20:31.291Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/4d/45/ba4c93527bc38616a8bd36488acb69a2212d60486794f0c1f318949bbb76/cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc", size = 211538, upload-time = "2026-08-03T21:20:33.808Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/b6ef565e452acb932fb0cb5443f44a78efbd1233e566f02b5a83855e9115/cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29", size = 210688, upload-time = "2026-08-03T21:20:34.974Z" }, + { url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/3b7176cb570a1d3e27faf67b72f591af508036e0d8b2be2ef9af9e8c84bb/cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4", size = 182868, upload-time = "2026-08-03T21:20:40.388Z" }, + { url = "https://files.pythonhosted.org/packages/56/78/31f00c1bcd97c9bbf55f1bfdf5bc809a5de8887473e90bb9960dca825e80/cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e", size = 194104, upload-time = "2026-08-03T21:20:41.725Z" }, + { url = "https://files.pythonhosted.org/packages/7b/1b/58496f2ed0a35de575250c02a43ab3cc2c04d494a88fed31c1cabc0fd176/cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5", size = 186402, upload-time = "2026-08-03T21:20:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/9ebe220eab48a093d1a5a5e339ab0dc7316eef3bb04d63c42f0251b61f50/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d", size = 194043, upload-time = "2026-08-03T21:20:48.179Z" }, + { url = "https://files.pythonhosted.org/packages/ff/69/844bad3ece306c4782c2ecb93597035b6690d48704b803914c199da1e8b3/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b", size = 196737, upload-time = "2026-08-03T21:20:49.457Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8a/af668013284634733f02d683458a0728739c7d6ddb5e14cb0c20832266fe/cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4", size = 184933, upload-time = "2026-08-03T21:20:50.639Z" }, + { url = "https://files.pythonhosted.org/packages/0c/75/2f5207ff6d1a613133b23a5203cc0c2a628313b5eb3974d7956ae3c57950/cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8", size = 185002, upload-time = "2026-08-03T21:20:52.173Z" }, + { url = "https://files.pythonhosted.org/packages/e2/31/9e1313b0a6e30e91b3b3d3fff51ae99c857c07738e3afcce1f7334e1b7ab/cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6", size = 222271, upload-time = "2026-08-03T21:20:53.462Z" }, + { url = "https://files.pythonhosted.org/packages/50/e3/f6234a833e6e08c7007003074723c406559eecf9b48dfc97471e5a8eb7a0/cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80", size = 209919, upload-time = "2026-08-03T21:20:54.783Z" }, + { url = "https://files.pythonhosted.org/packages/0d/fc/5f74e293fced6edb51af3a46c4ccf6c23c9943774ecb375ddbd522c76add/cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779", size = 208529, upload-time = "2026-08-03T21:20:56.066Z" }, + { url = "https://files.pythonhosted.org/packages/44/16/29e6d01b388bef055ecd6ca8244b3f4d336bd09e92d5d892187b9601084e/cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399", size = 221630, upload-time = "2026-08-03T21:20:57.336Z" }, + { url = "https://files.pythonhosted.org/packages/a4/18/fa7f1f6857d5eb88a4ca99ffcbfb7c387a287ccc154c64a73e86314745d7/cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688", size = 225134, upload-time = "2026-08-03T21:20:58.675Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9f/e8e3dfa04a1b4c241f8c91faacad872b4d4efd051d49764ad4e2fd4b9fea/cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7", size = 223197, upload-time = "2026-08-03T21:20:59.968Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7e/8debeb04f1ab9fe2a6963964cd6f1aaf7192627b83926586a6a4e089c9fa/cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac", size = 177683, upload-time = "2026-08-03T21:21:14.901Z" }, + { url = "https://files.pythonhosted.org/packages/e0/31/5158704cc474ab65c1647932e88be78dc0873f47130e253be38bcaf13d01/cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960", size = 187897, upload-time = "2026-08-03T21:21:16.108Z" }, + { url = "https://files.pythonhosted.org/packages/cc/4b/b3a2da8570c704ffc0f9762cdc3ec0f02c8573798e0b5cf7f11c82bbb70f/cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1", size = 182935, upload-time = "2026-08-03T21:21:17.271Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ef/5443574510a1207e6f6bc38ba6e1f1de36cb48fef07b2728bb896a21f430/cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc", size = 188464, upload-time = "2026-08-03T21:21:01.163Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ae/a56fa8c4686ad50e148fcbc8d3ae0d03915ff5c30d795058988c24118cef/cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab", size = 188262, upload-time = "2026-08-03T21:21:02.382Z" }, + { url = "https://files.pythonhosted.org/packages/53/b2/6187f46f2912276a3ae284076109cc5c8680482f11f766ccf26db4a86427/cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e", size = 223779, upload-time = "2026-08-03T21:21:03.553Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f6/c3ad28bd19f77047a03084424fbd4cbe997303267c14423737324be0385d/cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358", size = 211520, upload-time = "2026-08-03T21:21:04.863Z" }, + { url = "https://files.pythonhosted.org/packages/a0/cd/ccac9013a5bd9fd764de118674ab9c805b5ca10c19270d90ee273f8b2240/cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231", size = 210673, upload-time = "2026-08-03T21:21:06.223Z" }, + { url = "https://files.pythonhosted.org/packages/52/86/2976131c639aead931c5bee5aba67e4b09fbeb8018b6f282f70803f923a7/cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6", size = 223835, upload-time = "2026-08-03T21:21:07.539Z" }, + { url = "https://files.pythonhosted.org/packages/ac/0c/33a7aeab2f9c76918c52e084beb39c570db3588133412929e8ec06fab90b/cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94", size = 226705, upload-time = "2026-08-03T21:21:08.774Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/2cde30fdde421130bfc18f70395731a6e6b2053c6a1978a5258ff04e72fa/cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5", size = 225539, upload-time = "2026-08-03T21:21:09.911Z" }, + { url = "https://files.pythonhosted.org/packages/6d/cd/a361394c94b2129d604bb846f624a8e88255a3ee33129c434a00d715e64f/cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66", size = 182707, upload-time = "2026-08-03T21:21:11.226Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b5/ba2b299993c26577d529b6ae29841f9e15b9fcf004d65f423f4fcf94ade9/cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3", size = 193772, upload-time = "2026-08-03T21:21:12.39Z" }, + { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" }, + { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, + { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, + { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, + { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, + { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, + { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, + { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, + { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, + { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, + { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" }, + { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" }, + { url = "https://files.pythonhosted.org/packages/b2/06/97ec2aeae780b31d742b6352218b43841a6871e2564578ca522dce4a45c3/charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614", size = 317688, upload-time = "2026-07-07T14:33:35.408Z" }, + { url = "https://files.pythonhosted.org/packages/d0/39/8ff066c672434225f8d25f8b739f992af250944392173dcc88362681c9bf/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698", size = 214982, upload-time = "2026-07-07T14:33:36.996Z" }, + { url = "https://files.pythonhosted.org/packages/92/8f/3a47a3667c83c2df9483d91644c6c107de3bf8874aa1793da9d3012eb986/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b", size = 236460, upload-time = "2026-07-07T14:33:38.536Z" }, + { url = "https://files.pythonhosted.org/packages/f1/60/b22cdbee7e4013dab8b0d7647fc6181120fbbbc8f7025c226d15bd5a47fc/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9", size = 232003, upload-time = "2026-07-07T14:33:40.059Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f8/72eb13dcabe7257035cea8aefd922caad2f110d252bf9f67c4c2ca763aee/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33", size = 223149, upload-time = "2026-07-07T14:33:41.631Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3e/faee8f9de92b14ee1198e9163252bb15efee7301b31256a3b6d9ebfdd0dd/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63", size = 207901, upload-time = "2026-07-07T14:33:43.209Z" }, + { url = "https://files.pythonhosted.org/packages/3a/25/45f30093ae27dd7b92a793b61882a38685f993700113ca36e0c9c14965e1/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0", size = 219176, upload-time = "2026-07-07T14:33:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/48/18/c8f397329c35e32f6a837e488986f4ae03bd2abebc453b48714991630c2f/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe", size = 217356, upload-time = "2026-07-07T14:33:46.192Z" }, + { url = "https://files.pythonhosted.org/packages/86/7e/5ce0bba863470fd1902d5e5843968951bddf38abe4742fc97116ef4598b3/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35", size = 209614, upload-time = "2026-07-07T14:33:47.705Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ef/2473d3c4d869155be4af1191111d59c4d5c4e0173026f7e85b176e23bf65/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8", size = 224991, upload-time = "2026-07-07T14:33:49.238Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a3/53ddae3db108a088156aa8ddfafd411ebbc1340f48c5573f697b27f69a39/charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9", size = 150622, upload-time = "2026-07-07T14:33:50.711Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ef/6953a77c7cf2c2ff9998e6f575ab3e380119f100223381565a4f94c1f836/charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115", size = 161947, upload-time = "2026-07-07T14:33:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/6e/fb/d560d1d1555debbfe7849d9cac6145c1b537709d79576bf22557ed803b82/charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012", size = 152594, upload-time = "2026-07-07T14:33:53.486Z" }, + { url = "https://files.pythonhosted.org/packages/7e/8d/496817fa0944239ecae662dd57ea765cfeaec6a735f9f025d4b7b72e7143/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380", size = 317253, upload-time = "2026-07-07T14:33:54.994Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" }, + { url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload-time = "2026-07-07T14:33:59.811Z" }, + { url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload-time = "2026-07-07T14:34:01.517Z" }, + { url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload-time = "2026-07-07T14:34:03.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload-time = "2026-07-07T14:34:04.657Z" }, + { url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload-time = "2026-07-07T14:34:06.115Z" }, + { url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload-time = "2026-07-07T14:34:07.554Z" }, + { url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload-time = "2026-07-07T14:34:09.061Z" }, + { url = "https://files.pythonhosted.org/packages/9b/f2/c0d4b8508565a36bc5c624e88ed297f5b0b1095011034d7f5b83a69908b5/charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48", size = 151095, upload-time = "2026-07-07T14:34:10.901Z" }, + { url = "https://files.pythonhosted.org/packages/49/fd/a1d26144398c67486422a72bf5812cda22cb4ccfcd95a290fb41ceb4b8e2/charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b", size = 162796, upload-time = "2026-07-07T14:34:12.47Z" }, + { url = "https://files.pythonhosted.org/packages/20/95/d75e82f8ce9fd323ebf059c16c9aadefb22a1ecde13b7840b35835e4886c/charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519", size = 153334, upload-time = "2026-07-07T14:34:14.044Z" }, + { url = "https://files.pythonhosted.org/packages/00/5e/17398df3a139985ba9d11ed072531986f408c8fca952835ef1ab1820c02b/charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198", size = 338848, upload-time = "2026-07-07T14:34:15.688Z" }, + { url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload-time = "2026-07-07T14:34:17.248Z" }, + { url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload-time = "2026-07-07T14:34:18.813Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload-time = "2026-07-07T14:34:20.52Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload-time = "2026-07-07T14:34:22.189Z" }, + { url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload-time = "2026-07-07T14:34:23.857Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload-time = "2026-07-07T14:34:25.586Z" }, + { url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload-time = "2026-07-07T14:34:27.212Z" }, + { url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload-time = "2026-07-07T14:34:28.774Z" }, + { url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload-time = "2026-07-07T14:34:30.345Z" }, + { url = "https://files.pythonhosted.org/packages/b1/27/693ee5e8a18191eb38647360c51cd505013e2bd3b366aa43fd5344c21e3c/charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226", size = 155580, upload-time = "2026-07-07T14:34:31.884Z" }, + { url = "https://files.pythonhosted.org/packages/80/3f/bd97d3d9c613013d07cb7733d299385b41df37f0471310f5a73dc359f0b8/charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177", size = 167620, upload-time = "2026-07-07T14:34:33.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c6/eee9dca4439b1061f76373f06ea855678cc4a64c1c3c90b50e479edbb8eb/charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501", size = 158037, upload-time = "2026-07-07T14:34:35.018Z" }, + { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, +] + +[[package]] +name = "click" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "contourpy" +version = "1.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/45/adfee365d9ea3d853550b2e735f9d66366701c65db7855cd07621732ccfc/contourpy-1.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b08a32ea2f8e42cf1d4be3169a98dd4be32bafe4f22b6c4cb4ba810fa9e5d2cb", size = 293419, upload-time = "2025-07-26T12:01:21.16Z" }, + { url = "https://files.pythonhosted.org/packages/53/3e/405b59cfa13021a56bba395a6b3aca8cec012b45bf177b0eaf7a202cde2c/contourpy-1.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:556dba8fb6f5d8742f2923fe9457dbdd51e1049c4a43fd3986a0b14a1d815fc6", size = 273979, upload-time = "2025-07-26T12:01:22.448Z" }, + { url = "https://files.pythonhosted.org/packages/d4/1c/a12359b9b2ca3a845e8f7f9ac08bdf776114eb931392fcad91743e2ea17b/contourpy-1.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92d9abc807cf7d0e047b95ca5d957cf4792fcd04e920ca70d48add15c1a90ea7", size = 332653, upload-time = "2025-07-26T12:01:24.155Z" }, + { url = "https://files.pythonhosted.org/packages/63/12/897aeebfb475b7748ea67b61e045accdfcf0d971f8a588b67108ed7f5512/contourpy-1.3.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2e8faa0ed68cb29af51edd8e24798bb661eac3bd9f65420c1887b6ca89987c8", size = 379536, upload-time = "2025-07-26T12:01:25.91Z" }, + { url = "https://files.pythonhosted.org/packages/43/8a/a8c584b82deb248930ce069e71576fc09bd7174bbd35183b7943fb1064fd/contourpy-1.3.3-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:626d60935cf668e70a5ce6ff184fd713e9683fb458898e4249b63be9e28286ea", size = 384397, upload-time = "2025-07-26T12:01:27.152Z" }, + { url = "https://files.pythonhosted.org/packages/cc/8f/ec6289987824b29529d0dfda0d74a07cec60e54b9c92f3c9da4c0ac732de/contourpy-1.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1", size = 362601, upload-time = "2025-07-26T12:01:28.808Z" }, + { url = "https://files.pythonhosted.org/packages/05/0a/a3fe3be3ee2dceb3e615ebb4df97ae6f3828aa915d3e10549ce016302bd1/contourpy-1.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:451e71b5a7d597379ef572de31eeb909a87246974d960049a9848c3bc6c41bf7", size = 1331288, upload-time = "2025-07-26T12:01:31.198Z" }, + { url = "https://files.pythonhosted.org/packages/33/1d/acad9bd4e97f13f3e2b18a3977fe1b4a37ecf3d38d815333980c6c72e963/contourpy-1.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411", size = 1403386, upload-time = "2025-07-26T12:01:33.947Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8f/5847f44a7fddf859704217a99a23a4f6417b10e5ab1256a179264561540e/contourpy-1.3.3-cp312-cp312-win32.whl", hash = "sha256:023b44101dfe49d7d53932be418477dba359649246075c996866106da069af69", size = 185018, upload-time = "2025-07-26T12:01:35.64Z" }, + { url = "https://files.pythonhosted.org/packages/19/e8/6026ed58a64563186a9ee3f29f41261fd1828f527dd93d33b60feca63352/contourpy-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:8153b8bfc11e1e4d75bcb0bff1db232f9e10b274e0929de9d608027e0d34ff8b", size = 226567, upload-time = "2025-07-26T12:01:36.804Z" }, + { url = "https://files.pythonhosted.org/packages/d1/e2/f05240d2c39a1ed228d8328a78b6f44cd695f7ef47beb3e684cf93604f86/contourpy-1.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:07ce5ed73ecdc4a03ffe3e1b3e3c1166db35ae7584be76f65dbbe28a7791b0cc", size = 193655, upload-time = "2025-07-26T12:01:37.999Z" }, + { url = "https://files.pythonhosted.org/packages/68/35/0167aad910bbdb9599272bd96d01a9ec6852f36b9455cf2ca67bd4cc2d23/contourpy-1.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:177fb367556747a686509d6fef71d221a4b198a3905fe824430e5ea0fda54eb5", size = 293257, upload-time = "2025-07-26T12:01:39.367Z" }, + { url = "https://files.pythonhosted.org/packages/96/e4/7adcd9c8362745b2210728f209bfbcf7d91ba868a2c5f40d8b58f54c509b/contourpy-1.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d002b6f00d73d69333dac9d0b8d5e84d9724ff9ef044fd63c5986e62b7c9e1b1", size = 274034, upload-time = "2025-07-26T12:01:40.645Z" }, + { url = "https://files.pythonhosted.org/packages/73/23/90e31ceeed1de63058a02cb04b12f2de4b40e3bef5e082a7c18d9c8ae281/contourpy-1.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:348ac1f5d4f1d66d3322420f01d42e43122f43616e0f194fc1c9f5d830c5b286", size = 334672, upload-time = "2025-07-26T12:01:41.942Z" }, + { url = "https://files.pythonhosted.org/packages/ed/93/b43d8acbe67392e659e1d984700e79eb67e2acb2bd7f62012b583a7f1b55/contourpy-1.3.3-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:655456777ff65c2c548b7c454af9c6f33f16c8884f11083244b5819cc214f1b5", size = 381234, upload-time = "2025-07-26T12:01:43.499Z" }, + { url = "https://files.pythonhosted.org/packages/46/3b/bec82a3ea06f66711520f75a40c8fc0b113b2a75edb36aa633eb11c4f50f/contourpy-1.3.3-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:644a6853d15b2512d67881586bd03f462c7ab755db95f16f14d7e238f2852c67", size = 385169, upload-time = "2025-07-26T12:01:45.219Z" }, + { url = "https://files.pythonhosted.org/packages/4b/32/e0f13a1c5b0f8572d0ec6ae2f6c677b7991fafd95da523159c19eff0696a/contourpy-1.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4debd64f124ca62069f313a9cb86656ff087786016d76927ae2cf37846b006c9", size = 362859, upload-time = "2025-07-26T12:01:46.519Z" }, + { url = "https://files.pythonhosted.org/packages/33/71/e2a7945b7de4e58af42d708a219f3b2f4cff7386e6b6ab0a0fa0033c49a9/contourpy-1.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a15459b0f4615b00bbd1e91f1b9e19b7e63aea7483d03d804186f278c0af2659", size = 1332062, upload-time = "2025-07-26T12:01:48.964Z" }, + { url = "https://files.pythonhosted.org/packages/12/fc/4e87ac754220ccc0e807284f88e943d6d43b43843614f0a8afa469801db0/contourpy-1.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca0fdcd73925568ca027e0b17ab07aad764be4706d0a925b89227e447d9737b7", size = 1403932, upload-time = "2025-07-26T12:01:51.979Z" }, + { url = "https://files.pythonhosted.org/packages/a6/2e/adc197a37443f934594112222ac1aa7dc9a98faf9c3842884df9a9d8751d/contourpy-1.3.3-cp313-cp313-win32.whl", hash = "sha256:b20c7c9a3bf701366556e1b1984ed2d0cedf999903c51311417cf5f591d8c78d", size = 185024, upload-time = "2025-07-26T12:01:53.245Z" }, + { url = "https://files.pythonhosted.org/packages/18/0b/0098c214843213759692cc638fce7de5c289200a830e5035d1791d7a2338/contourpy-1.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:1cadd8b8969f060ba45ed7c1b714fe69185812ab43bd6b86a9123fe8f99c3263", size = 226578, upload-time = "2025-07-26T12:01:54.422Z" }, + { url = "https://files.pythonhosted.org/packages/8a/9a/2f6024a0c5995243cd63afdeb3651c984f0d2bc727fd98066d40e141ad73/contourpy-1.3.3-cp313-cp313-win_arm64.whl", hash = "sha256:fd914713266421b7536de2bfa8181aa8c699432b6763a0ea64195ebe28bff6a9", size = 193524, upload-time = "2025-07-26T12:01:55.73Z" }, + { url = "https://files.pythonhosted.org/packages/c0/b3/f8a1a86bd3298513f500e5b1f5fd92b69896449f6cab6a146a5d52715479/contourpy-1.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:88df9880d507169449d434c293467418b9f6cbe82edd19284aa0409e7fdb933d", size = 306730, upload-time = "2025-07-26T12:01:57.051Z" }, + { url = "https://files.pythonhosted.org/packages/3f/11/4780db94ae62fc0c2053909b65dc3246bd7cecfc4f8a20d957ad43aa4ad8/contourpy-1.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:d06bb1f751ba5d417047db62bca3c8fde202b8c11fb50742ab3ab962c81e8216", size = 287897, upload-time = "2025-07-26T12:01:58.663Z" }, + { url = "https://files.pythonhosted.org/packages/ae/15/e59f5f3ffdd6f3d4daa3e47114c53daabcb18574a26c21f03dc9e4e42ff0/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e4e6b05a45525357e382909a4c1600444e2a45b4795163d3b22669285591c1ae", size = 326751, upload-time = "2025-07-26T12:02:00.343Z" }, + { url = "https://files.pythonhosted.org/packages/0f/81/03b45cfad088e4770b1dcf72ea78d3802d04200009fb364d18a493857210/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab3074b48c4e2cf1a960e6bbeb7f04566bf36b1861d5c9d4d8ac04b82e38ba20", size = 375486, upload-time = "2025-07-26T12:02:02.128Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ba/49923366492ffbdd4486e970d421b289a670ae8cf539c1ea9a09822b371a/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6c3d53c796f8647d6deb1abe867daeb66dcc8a97e8455efa729516b997b8ed99", size = 388106, upload-time = "2025-07-26T12:02:03.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/52/5b00ea89525f8f143651f9f03a0df371d3cbd2fccd21ca9b768c7a6500c2/contourpy-1.3.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50ed930df7289ff2a8d7afeb9603f8289e5704755c7e5c3bbd929c90c817164b", size = 352548, upload-time = "2025-07-26T12:02:05.165Z" }, + { url = "https://files.pythonhosted.org/packages/32/1d/a209ec1a3a3452d490f6b14dd92e72280c99ae3d1e73da74f8277d4ee08f/contourpy-1.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4feffb6537d64b84877da813a5c30f1422ea5739566abf0bd18065ac040e120a", size = 1322297, upload-time = "2025-07-26T12:02:07.379Z" }, + { url = "https://files.pythonhosted.org/packages/bc/9e/46f0e8ebdd884ca0e8877e46a3f4e633f6c9c8c4f3f6e72be3fe075994aa/contourpy-1.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2b7e9480ffe2b0cd2e787e4df64270e3a0440d9db8dc823312e2c940c167df7e", size = 1391023, upload-time = "2025-07-26T12:02:10.171Z" }, + { url = "https://files.pythonhosted.org/packages/b9/70/f308384a3ae9cd2209e0849f33c913f658d3326900d0ff5d378d6a1422d2/contourpy-1.3.3-cp313-cp313t-win32.whl", hash = "sha256:283edd842a01e3dcd435b1c5116798d661378d83d36d337b8dde1d16a5fc9ba3", size = 196157, upload-time = "2025-07-26T12:02:11.488Z" }, + { url = "https://files.pythonhosted.org/packages/b2/dd/880f890a6663b84d9e34a6f88cded89d78f0091e0045a284427cb6b18521/contourpy-1.3.3-cp313-cp313t-win_amd64.whl", hash = "sha256:87acf5963fc2b34825e5b6b048f40e3635dd547f590b04d2ab317c2619ef7ae8", size = 240570, upload-time = "2025-07-26T12:02:12.754Z" }, + { url = "https://files.pythonhosted.org/packages/80/99/2adc7d8ffead633234817ef8e9a87115c8a11927a94478f6bb3d3f4d4f7d/contourpy-1.3.3-cp313-cp313t-win_arm64.whl", hash = "sha256:3c30273eb2a55024ff31ba7d052dde990d7d8e5450f4bbb6e913558b3d6c2301", size = 199713, upload-time = "2025-07-26T12:02:14.4Z" }, + { url = "https://files.pythonhosted.org/packages/72/8b/4546f3ab60f78c514ffb7d01a0bd743f90de36f0019d1be84d0a708a580a/contourpy-1.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fde6c716d51c04b1c25d0b90364d0be954624a0ee9d60e23e850e8d48353d07a", size = 292189, upload-time = "2025-07-26T12:02:16.095Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e1/3542a9cb596cadd76fcef413f19c79216e002623158befe6daa03dbfa88c/contourpy-1.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cbedb772ed74ff5be440fa8eee9bd49f64f6e3fc09436d9c7d8f1c287b121d77", size = 273251, upload-time = "2025-07-26T12:02:17.524Z" }, + { url = "https://files.pythonhosted.org/packages/b1/71/f93e1e9471d189f79d0ce2497007731c1e6bf9ef6d1d61b911430c3db4e5/contourpy-1.3.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22e9b1bd7a9b1d652cd77388465dc358dafcd2e217d35552424aa4f996f524f5", size = 335810, upload-time = "2025-07-26T12:02:18.9Z" }, + { url = "https://files.pythonhosted.org/packages/91/f9/e35f4c1c93f9275d4e38681a80506b5510e9327350c51f8d4a5a724d178c/contourpy-1.3.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a22738912262aa3e254e4f3cb079a95a67132fc5a063890e224393596902f5a4", size = 382871, upload-time = "2025-07-26T12:02:20.418Z" }, + { url = "https://files.pythonhosted.org/packages/b5/71/47b512f936f66a0a900d81c396a7e60d73419868fba959c61efed7a8ab46/contourpy-1.3.3-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:afe5a512f31ee6bd7d0dda52ec9864c984ca3d66664444f2d72e0dc4eb832e36", size = 386264, upload-time = "2025-07-26T12:02:21.916Z" }, + { url = "https://files.pythonhosted.org/packages/04/5f/9ff93450ba96b09c7c2b3f81c94de31c89f92292f1380261bd7195bea4ea/contourpy-1.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f64836de09927cba6f79dcd00fdd7d5329f3fccc633468507079c829ca4db4e3", size = 363819, upload-time = "2025-07-26T12:02:23.759Z" }, + { url = "https://files.pythonhosted.org/packages/3e/a6/0b185d4cc480ee494945cde102cb0149ae830b5fa17bf855b95f2e70ad13/contourpy-1.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1fd43c3be4c8e5fd6e4f2baeae35ae18176cf2e5cced681cca908addf1cdd53b", size = 1333650, upload-time = "2025-07-26T12:02:26.181Z" }, + { url = "https://files.pythonhosted.org/packages/43/d7/afdc95580ca56f30fbcd3060250f66cedbde69b4547028863abd8aa3b47e/contourpy-1.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6afc576f7b33cf00996e5c1102dc2a8f7cc89e39c0b55df93a0b78c1bd992b36", size = 1404833, upload-time = "2025-07-26T12:02:28.782Z" }, + { url = "https://files.pythonhosted.org/packages/e2/e2/366af18a6d386f41132a48f033cbd2102e9b0cf6345d35ff0826cd984566/contourpy-1.3.3-cp314-cp314-win32.whl", hash = "sha256:66c8a43a4f7b8df8b71ee1840e4211a3c8d93b214b213f590e18a1beca458f7d", size = 189692, upload-time = "2025-07-26T12:02:30.128Z" }, + { url = "https://files.pythonhosted.org/packages/7d/c2/57f54b03d0f22d4044b8afb9ca0e184f8b1afd57b4f735c2fa70883dc601/contourpy-1.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:cf9022ef053f2694e31d630feaacb21ea24224be1c3ad0520b13d844274614fd", size = 232424, upload-time = "2025-07-26T12:02:31.395Z" }, + { url = "https://files.pythonhosted.org/packages/18/79/a9416650df9b525737ab521aa181ccc42d56016d2123ddcb7b58e926a42c/contourpy-1.3.3-cp314-cp314-win_arm64.whl", hash = "sha256:95b181891b4c71de4bb404c6621e7e2390745f887f2a026b2d99e92c17892339", size = 198300, upload-time = "2025-07-26T12:02:32.956Z" }, + { url = "https://files.pythonhosted.org/packages/1f/42/38c159a7d0f2b7b9c04c64ab317042bb6952b713ba875c1681529a2932fe/contourpy-1.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:33c82d0138c0a062380332c861387650c82e4cf1747aaa6938b9b6516762e772", size = 306769, upload-time = "2025-07-26T12:02:34.2Z" }, + { url = "https://files.pythonhosted.org/packages/c3/6c/26a8205f24bca10974e77460de68d3d7c63e282e23782f1239f226fcae6f/contourpy-1.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ea37e7b45949df430fe649e5de8351c423430046a2af20b1c1961cae3afcda77", size = 287892, upload-time = "2025-07-26T12:02:35.807Z" }, + { url = "https://files.pythonhosted.org/packages/66/06/8a475c8ab718ebfd7925661747dbb3c3ee9c82ac834ccb3570be49d129f4/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d304906ecc71672e9c89e87c4675dc5c2645e1f4269a5063b99b0bb29f232d13", size = 326748, upload-time = "2025-07-26T12:02:37.193Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a3/c5ca9f010a44c223f098fccd8b158bb1cb287378a31ac141f04730dc49be/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ca658cd1a680a5c9ea96dc61cdbae1e85c8f25849843aa799dfd3cb370ad4fbe", size = 375554, upload-time = "2025-07-26T12:02:38.894Z" }, + { url = "https://files.pythonhosted.org/packages/80/5b/68bd33ae63fac658a4145088c1e894405e07584a316738710b636c6d0333/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ab2fd90904c503739a75b7c8c5c01160130ba67944a7b77bbf36ef8054576e7f", size = 388118, upload-time = "2025-07-26T12:02:40.642Z" }, + { url = "https://files.pythonhosted.org/packages/40/52/4c285a6435940ae25d7410a6c36bda5145839bc3f0beb20c707cda18b9d2/contourpy-1.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7301b89040075c30e5768810bc96a8e8d78085b47d8be6e4c3f5a0b4ed478a0", size = 352555, upload-time = "2025-07-26T12:02:42.25Z" }, + { url = "https://files.pythonhosted.org/packages/24/ee/3e81e1dd174f5c7fefe50e85d0892de05ca4e26ef1c9a59c2a57e43b865a/contourpy-1.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2a2a8b627d5cc6b7c41a4beff6c5ad5eb848c88255fda4a8745f7e901b32d8e4", size = 1322295, upload-time = "2025-07-26T12:02:44.668Z" }, + { url = "https://files.pythonhosted.org/packages/3c/b2/6d913d4d04e14379de429057cd169e5e00f6c2af3bb13e1710bcbdb5da12/contourpy-1.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fd6ec6be509c787f1caf6b247f0b1ca598bef13f4ddeaa126b7658215529ba0f", size = 1391027, upload-time = "2025-07-26T12:02:47.09Z" }, + { url = "https://files.pythonhosted.org/packages/93/8a/68a4ec5c55a2971213d29a9374913f7e9f18581945a7a31d1a39b5d2dfe5/contourpy-1.3.3-cp314-cp314t-win32.whl", hash = "sha256:e74a9a0f5e3fff48fb5a7f2fd2b9b70a3fe014a67522f79b7cca4c0c7e43c9ae", size = 202428, upload-time = "2025-07-26T12:02:48.691Z" }, + { url = "https://files.pythonhosted.org/packages/fa/96/fd9f641ffedc4fa3ace923af73b9d07e869496c9cc7a459103e6e978992f/contourpy-1.3.3-cp314-cp314t-win_amd64.whl", hash = "sha256:13b68d6a62db8eafaebb8039218921399baf6e47bf85006fd8529f2a08ef33fc", size = 250331, upload-time = "2025-07-26T12:02:50.137Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8c/469afb6465b853afff216f9528ffda78a915ff880ed58813ba4faf4ba0b6/contourpy-1.3.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b7448cb5a725bb1e35ce88771b86fba35ef418952474492cf7c764059933ff8b", size = 203831, upload-time = "2025-07-26T12:02:51.449Z" }, +] + +[[package]] +name = "cryptography" +version = "50.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/41/6cbdcf9142d00fe82836fbb51e503e58088575cf7a0fe1dbff6695bf0840/cryptography-50.0.0.tar.gz", hash = "sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9", size = 880201, upload-time = "2026-07-31T14:25:10.11Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/5c/59086b4aac5e879d38ddbcf74e4be7ade89cebc3eb199a55da998c3bb46a/cryptography-50.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:031e2d5dd4bb9caa3ca9c82e5a197fd8ae680232cee62603d1a813f3f07e3d03", size = 4001252, upload-time = "2026-07-31T14:23:33.331Z" }, + { url = "https://files.pythonhosted.org/packages/57/ef/8f2df13c7216bcad3e1c74e07f6e193d93e998e114f524a53877c9af27ad/cryptography-50.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fd9192b7b70c573d7f214eb1ae35e00d359f6f5e4b27c7e21e30de1fc6204645", size = 4719554, upload-time = "2026-07-31T14:23:35.611Z" }, + { url = "https://files.pythonhosted.org/packages/d9/41/029086c34d91052fc3b88bcc8056f709a7c915c7a23b235a54eb800b1c97/cryptography-50.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:06a32a980526a6ab9a4b9bf8f7385800791e2bb960903cb6b530e4817509a3b7", size = 4702130, upload-time = "2026-07-31T14:23:37.635Z" }, + { url = "https://files.pythonhosted.org/packages/7d/ff/b6ce0954962e7f7b969f850a883744197bb3910bdfd7b6da162eab7d9f68/cryptography-50.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a1b30560f2acc95aa8b2e06e716a13dbfc97314747b80d9707e307f77b40d6b3", size = 4725244, upload-time = "2026-07-31T14:23:39.471Z" }, + { url = "https://files.pythonhosted.org/packages/06/1e/63a1027cb7fec360a182208e1b7767d5aa1fe57be3d6aa856e69a321edc0/cryptography-50.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:8d89f3976b10b4ce31118de72329025f70d2c6ead14a8217c5514dd2c6d5a78f", size = 5342265, upload-time = "2026-07-31T14:23:41.286Z" }, + { url = "https://files.pythonhosted.org/packages/6b/72/a1116d683a6d7ece94590013882515de087edf9ef0e6292aae615a44df73/cryptography-50.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b42a28c1844fd9de8f3f7d540e36b66f3a9c83fceac7170ebc7a6a19edd9dcae", size = 4734609, upload-time = "2026-07-31T14:23:43.139Z" }, + { url = "https://files.pythonhosted.org/packages/15/37/36a9c479bbe49acea2636c7fd3360d20f7b7e079c300352011c44850b181/cryptography-50.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:900131fafd8aead39ac7dd3a7e833be754c17a95cfd91221636949fe4eb0aa8a", size = 4356517, upload-time = "2026-07-31T14:23:44.939Z" }, + { url = "https://files.pythonhosted.org/packages/32/98/8a151d64367204cbc63ec65d37502f1d9c53cf4bfc6ec3c532614dbec60d/cryptography-50.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:07949c449a1abcf60d1ee6e88956d89404c7df3c8258f46589e912988e551987", size = 4724529, upload-time = "2026-07-31T14:23:46.93Z" }, + { url = "https://files.pythonhosted.org/packages/22/f6/ec13b470172126464a86bf54d2294a46d29837fc51ba3e45d4047946fb5e/cryptography-50.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f89831ef99dd7dd169ab06d63a831adb9e20a87aac6d380266bbda5823349169", size = 5299852, upload-time = "2026-07-31T14:23:48.851Z" }, + { url = "https://files.pythonhosted.org/packages/da/3a/f05e32c99d440c9bb891ea0e36c9091891e36be5a9a87ab2ee6ea20729f6/cryptography-50.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:82148ec5bddac30b51a5b3c1945075f896fa022cb93f8e4a01e9f6ee95292c5f", size = 4734462, upload-time = "2026-07-31T14:23:50.861Z" }, + { url = "https://files.pythonhosted.org/packages/ca/dc/bd72b26be8953f80625f63151efd38eee71c76ca6cf591c08ff34615a79e/cryptography-50.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1489e263a8048bb8b6a8bac662eb2d402ea5d2b7b4699b72f385f1e2772db105", size = 4852708, upload-time = "2026-07-31T14:23:52.715Z" }, + { url = "https://files.pythonhosted.org/packages/27/20/c930314a2ab476d15dec966ec87e2e9637bb02b06106b12c0396c57bb603/cryptography-50.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7cec5b856506da6defb290f30c9ee687d5f5e8cb0bd3f6459dde43b0b4fa40ef", size = 5004179, upload-time = "2026-07-31T14:23:54.887Z" }, + { url = "https://files.pythonhosted.org/packages/32/2e/c9db68a0c4bfa28e310707527c0ee3a2bd254104d2e02e68f368e197aa4c/cryptography-50.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:bd1c592e4d5974f0d08d4888e432157adba757c66da0246918e43677fafa2d30", size = 3840395, upload-time = "2026-07-31T14:23:56.677Z" }, + { url = "https://files.pythonhosted.org/packages/c3/fb/951032a3bf22a5697c83183fb6294a4843772947a70e616c57b3ff5f522e/cryptography-50.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:49e7d93abdbd2990caced757e5fade25302f719c3c8fb6e6fff2dde98999fc41", size = 3989258, upload-time = "2026-07-31T14:23:58.881Z" }, + { url = "https://files.pythonhosted.org/packages/d4/67/91eb047e69c5e845f2f14b8a2e4a1aab0f283cb885531e9e22c8adb176bc/cryptography-50.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:19736989797678c6af1e55cd49055cdbcb55d8f6b5583ac5335f933aba9101dc", size = 4700648, upload-time = "2026-07-31T14:24:00.702Z" }, + { url = "https://files.pythonhosted.org/packages/30/82/85f0f7425c856b9f96459411eb12e74ef72df9caf6f8f15bf23a33ff131f/cryptography-50.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:80b63928fa35083b33966ce1efb70e5b9607181e49dcd1c22c8c005e319f667f", size = 4682442, upload-time = "2026-07-31T14:24:02.538Z" }, + { url = "https://files.pythonhosted.org/packages/1a/28/b555a365adff1cca2fbe7b9e487d68a40de6bc67ff2cb587473eb43de0e7/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:d58c3db7cd6eed54e6c06744db55456b65ebd7492ddeae9c1e93cfca7aa857d3", size = 4707596, upload-time = "2026-07-31T14:24:04.394Z" }, + { url = "https://files.pythonhosted.org/packages/72/d8/f52538140cc719df62a01cf87d1c7142318d235817109d6f4054d7c352d6/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:df2a58a472f332225671c35b0a830208b86d004f82baa8530fa3782c85646533", size = 5314552, upload-time = "2026-07-31T14:24:06.31Z" }, + { url = "https://files.pythonhosted.org/packages/38/14/6120e5bd7c5aa022ad15424ba4d5c5269d0d9448ed4d55e492ea91e3c1c4/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:11b74db56cdbe3cdee6e3f6982ecb70334fa10dce99ed58bf7894aaaa3b2a037", size = 4717113, upload-time = "2026-07-31T14:24:08.349Z" }, + { url = "https://files.pythonhosted.org/packages/fa/71/190bf38c3ee2e0f8efc9860ae100c9df4169742eef274b91e7aa1cb133b9/cryptography-50.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f59e38625469987d7ef6d495323c55e7db6c212eaf6112267e0d3b565a2e9c9f", size = 4338580, upload-time = "2026-07-31T14:24:10.227Z" }, + { url = "https://files.pythonhosted.org/packages/3a/63/504ccfbbe61fd8aa983f7f146399cdf034c72c2fc55f5b2dfdcdcdb20c99/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ecfed7367f965a0328cfbdd70da860f15441f002f613185668c6e6ebf5a0ac11", size = 4707038, upload-time = "2026-07-31T14:24:12.169Z" }, + { url = "https://files.pythonhosted.org/packages/01/77/2cf79bbfc4d12ca106437a6e170d6aaa01a373e93093118aaaef0e801bd4/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:9aa87839c383bdbab6ef865787a1fb877af8dd03464c4400322726feaaadfc6d", size = 5273110, upload-time = "2026-07-31T14:24:14.38Z" }, + { url = "https://files.pythonhosted.org/packages/e5/45/8aae2972c520145377ea3559a605a899bebe227bf070b33cdb445929a9b9/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:6ba6a53445bd3cfa809ef3ef5f1589aa6ba08784a1d962bf47d0940e871dab1c", size = 4716439, upload-time = "2026-07-31T14:24:16.415Z" }, + { url = "https://files.pythonhosted.org/packages/7b/20/4fe50b619a48c2525cc46e2dbc1ac490708d704be5d467bdaac6dc955682/cryptography-50.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3f5735ffe4996d28b809371756219f5354864902a3b9e7c0b9ee87041209fc9c", size = 4837383, upload-time = "2026-07-31T14:24:18.553Z" }, + { url = "https://files.pythonhosted.org/packages/92/91/3a31366e183343d3703f8995c095f5734676bd6938118047e50fcf279eb4/cryptography-50.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1b4a266766514614f8aa60416e71f2fc6e575d36e7bdc90f644fadb2f4b75b95", size = 4985772, upload-time = "2026-07-31T14:24:20.385Z" }, + { url = "https://files.pythonhosted.org/packages/74/9a/02ffe35b2853d121689871eb5dce862092562b3a1ed5cc98f1aaed441506/cryptography-50.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:12b9c6996425c76ea6c457ace4f3073e715b8c545add07cd1a8f3a4f90691269", size = 3816291, upload-time = "2026-07-31T14:24:22.125Z" }, + { url = "https://files.pythonhosted.org/packages/03/37/73d005be173aff344af30e9fd2a576575cb2391a7101d9cd3842e1fa8cce/cryptography-50.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ccdc4a71a4dabae05de219404f9f4abc38e3b58422177ff93d0da05967dafa07", size = 4036009, upload-time = "2026-07-31T14:24:24.122Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c6/7a6202a534e32103a285b7834a120869557fe198d51d7cfe59754c8bda9c/cryptography-50.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:910e1d2668e7de9648f2bcee30e180db2a6b15c30f887d7c4c93ddf96e3992e3", size = 4745252, upload-time = "2026-07-31T14:24:26.118Z" }, + { url = "https://files.pythonhosted.org/packages/85/4f/0fa8c2f4428198f15d9ff8d63400e27afbf94ce833f6108da1eb3753f945/cryptography-50.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a91296cb61e8df6f86d0c19cc4068228da256bf59bf86049fbd821084565327f", size = 4728939, upload-time = "2026-07-31T14:24:27.994Z" }, + { url = "https://files.pythonhosted.org/packages/d1/63/54dd723490ba2dc09b299682c10b38db38f159728bcaae8c591b8af2f22d/cryptography-50.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e722f16708d854fe924790e051061f6704a472c3bac347b6fd88033ea8dd0dc5", size = 4748483, upload-time = "2026-07-31T14:24:30.254Z" }, + { url = "https://files.pythonhosted.org/packages/1d/dd/7c77d26285cc7f6991efce64a0f5b4f9383bfa5dd8c5033003eaf7db4cdb/cryptography-50.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:d764dcf130c428ef66786f866dd750f53182bc608813489915e9fc106bb0c82f", size = 5367599, upload-time = "2026-07-31T14:24:32.457Z" }, + { url = "https://files.pythonhosted.org/packages/46/c9/f60aed34c013f317f92817b6c171c2d22a78270fa41109bd4b08af26b194/cryptography-50.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:105110f43a471dbd0060b9c9516cb8a6a79233631a04cc2ba16f28323ac6e025", size = 4762647, upload-time = "2026-07-31T14:24:34.599Z" }, + { url = "https://files.pythonhosted.org/packages/be/f3/f9a0173b139372c3a48ed98154b45cc6b9de17c789d5ab552e621c293609/cryptography-50.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:828743d939e9629bc267b8e2d08d8bb67cd4319c771a33d4b18b22dd8fb7440a", size = 4385197, upload-time = "2026-07-31T14:24:36.647Z" }, + { url = "https://files.pythonhosted.org/packages/d8/36/83bb81f6e569bc38e1e4a7bc80f29b46bb9601920bc455fc8e888f5d5742/cryptography-50.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:2a8183b489dc1f7f80f135780fadc1108f14b31b8a40411c7a5b17425f65f28b", size = 4748095, upload-time = "2026-07-31T14:24:39.493Z" }, + { url = "https://files.pythonhosted.org/packages/6b/16/d3008eff98c764979865834c3d386d4fd041b5f52e7f34fc29ac1a5eb515/cryptography-50.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6e7d61120573a7f2cd94cc095f9e81f6967c61ccdf194285aa143ecec8e0b708", size = 5325948, upload-time = "2026-07-31T14:24:41.556Z" }, + { url = "https://files.pythonhosted.org/packages/9c/f8/d97f9603efda3888187bfdb893f26c41be4735c10631d05d284ee6b047c4/cryptography-50.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:37fdb0d0111f1e2ff07139dfb79f1b49531f8e213c46f1163dd7642979b58c47", size = 4762400, upload-time = "2026-07-31T14:24:43.636Z" }, + { url = "https://files.pythonhosted.org/packages/64/a2/4615c8f7d81a00b1d6e6afe19f694e1543582349fb5f4076f6cb5dc36485/cryptography-50.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87f62a3d3b9888ed0fdde100ec06aa61ca9cd44bad9057d1dff9a516b5f5bb9", size = 4878208, upload-time = "2026-07-31T14:24:45.522Z" }, + { url = "https://files.pythonhosted.org/packages/d2/1a/efcfb02f91407149a0dacffffab791f7e19bf6385f63b3666dc8b5e5c9c8/cryptography-50.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:65c2c3add92b45fd0709db8594536aea39c2a67af0e27ffcf049c498501140b7", size = 5037050, upload-time = "2026-07-31T14:24:47.697Z" }, + { url = "https://files.pythonhosted.org/packages/57/30/4a22984d4f1bdfb8c054f07a92bc176b97a3134cc1d6c4b3bffb1f3688b4/cryptography-50.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:d24fead1d4d076e1bfb006dcec392074a3cd8d7b4fc8a595aa64073b2b7a96ba", size = 3874135, upload-time = "2026-07-31T14:24:50.085Z" }, +] + +[[package]] +name = "cycler" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/95/a3dbbb5028f35eafb79008e7522a75244477d2838f38cbb722248dabc2a8/cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c", size = 7615, upload-time = "2023-10-07T05:32:18.335Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, +] + +[[package]] +name = "datasets" +version = "5.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill" }, + { name = "filelock" }, + { name = "fsspec", extra = ["http"] }, + { name = "httpx" }, + { name = "huggingface-hub" }, + { name = "multiprocess" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "xxhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0a/5b/836516269d4f618efe621661cfb6f9acc57e6f95265db3efaee48a5ffe04/datasets-5.0.1.tar.gz", hash = "sha256:ce22bb851efd7494f08aad33b940803784434f6e77763d00679a0dc45fcf686a", size = 641498, upload-time = "2026-07-28T11:09:12.016Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/0b/98fc6eb83333508ca5f44c52b3e287ea8137a0ad582714e2cbc67a02154b/datasets-5.0.1-py3-none-any.whl", hash = "sha256:9fbf73688f8c18f7529b4fe592abd04015f81d1e58001e4bac73ffb2b39d7cc4", size = 559079, upload-time = "2026-07-28T11:09:10.266Z" }, +] + +[[package]] +name = "decorator" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/60/8b/32f9823da46cde7df2087faa08cd98d01b908f8dcab982cdba9c84e85355/decorator-5.3.1.tar.gz", hash = "sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82", size = 58084, upload-time = "2026-05-18T06:03:28.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/7f/798705f5296a58ca505d600456748d1be48078eac8a7050d8a98bc9edb89/decorator-5.3.1-py3-none-any.whl", hash = "sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c", size = 10365, upload-time = "2026-05-18T06:03:26.517Z" }, +] + +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, +] + +[[package]] +name = "dill" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/81/e1/56027a71e31b02ddc53c7d65b01e68edf64dea2932122fe7746a516f75d5/dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa", size = 187315, upload-time = "2026-01-19T02:36:56.85Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, +] + +[[package]] +name = "distance" +version = "0.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/1a/883e47df323437aefa0d0a92ccfb38895d9416bd0b56262c2e46a47767b8/Distance-0.1.3.tar.gz", hash = "sha256:60807584f5b6003f5c521aa73f39f51f631de3be5cccc5a1d67166fcbf0d4551", size = 180271, upload-time = "2013-11-21T00:14:34.152Z" } + +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + +[[package]] +name = "editdistance" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/18/9f4f975ca87a390832b1c22478f3702fcdf739f83211e24d054b7551270d/editdistance-0.8.1.tar.gz", hash = "sha256:d1cdf80a5d5014b0c9126a69a42ce55a457b457f6986ff69ca98e4fe4d2d8fed", size = 50006, upload-time = "2024-02-10T07:44:53.914Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/4c/7f195588949b4e72436dc7fc902632381f96e586af829685b56daebb38b8/editdistance-0.8.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b04af61b3fcdd287a07c15b6ae3b02af01c5e3e9c3aca76b8c1d13bd266b6f57", size = 106723, upload-time = "2024-02-10T07:43:50.268Z" }, + { url = "https://files.pythonhosted.org/packages/8d/82/31dc1640d830cd7d36865098329f34e4dad3b77f31cfb9404b347e700196/editdistance-0.8.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:18fc8b6eaae01bfd9cf999af726c1e8dcf667d120e81aa7dbd515bea7427f62f", size = 80998, upload-time = "2024-02-10T07:43:51.259Z" }, + { url = "https://files.pythonhosted.org/packages/ea/2a/6b823e71cef694d6f070a1d82be2842706fa193541aab8856a8f42044cd0/editdistance-0.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6a87839450a5987028738d061ffa5ef6a68bac2ddc68c9147a8aae9806629c7f", size = 79248, upload-time = "2024-02-10T07:43:52.873Z" }, + { url = "https://files.pythonhosted.org/packages/e1/31/bfb8e590f922089dc3471ed7828a6da2fc9453eba38c332efa9ee8749fd7/editdistance-0.8.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24b5f9c9673c823d91b5973d0af8b39f883f414a55ade2b9d097138acd10f31e", size = 415262, upload-time = "2024-02-10T07:43:54.498Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c7/57423942b2f847cdbbb46494568d00cd8a45500904ea026f0aad6ca01bc7/editdistance-0.8.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c59248eabfad603f0fba47b0c263d5dc728fb01c2b6b50fb6ca187cec547fdb3", size = 418905, upload-time = "2024-02-10T07:43:55.779Z" }, + { url = "https://files.pythonhosted.org/packages/1b/05/dfa4cdcce063596cbf0d7a32c46cd0f4fa70980311b7da64d35f33ad02a0/editdistance-0.8.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84e239d88ff52821cf64023fabd06a1d9a07654f364b64bf1284577fd3a79d0e", size = 412511, upload-time = "2024-02-10T07:43:57.567Z" }, + { url = "https://files.pythonhosted.org/packages/0e/14/39608ff724a9523f187c4e28926d78bc68f2798f74777ac6757981108345/editdistance-0.8.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2f7f71698f83e8c83839ac0d876a0f4ef996c86c5460aebd26d85568d4afd0db", size = 917293, upload-time = "2024-02-10T07:43:59.559Z" }, + { url = "https://files.pythonhosted.org/packages/df/92/4a1c61d72da40dedfd0ff950fdc71ae83f478330c58a8bccfd776518bd67/editdistance-0.8.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:04e229d6f4ce0c12abc9f4cd4023a5b5fa9620226e0207b119c3c2778b036250", size = 975580, upload-time = "2024-02-10T07:44:01.328Z" }, + { url = "https://files.pythonhosted.org/packages/47/3d/9877566e724c8a37f2228a84ec5cbf66dbfd0673515baf68a0fe07caff40/editdistance-0.8.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e16721636da6d6b68a2c09eaced35a94f4a4a704ec09f45756d4fd5e128ed18d", size = 929121, upload-time = "2024-02-10T07:44:02.764Z" }, + { url = "https://files.pythonhosted.org/packages/d2/f5/8c50757d198b8ca30ddb91e8b8f0247a8dca04ff2ec30755245f0ab1ff0c/editdistance-0.8.1-cp312-cp312-win32.whl", hash = "sha256:87533cf2ebc3777088d991947274cd7e1014b9c861a8aa65257bcdc0ee492526", size = 81039, upload-time = "2024-02-10T07:44:04.134Z" }, + { url = "https://files.pythonhosted.org/packages/28/f0/65101e51dc7c850e7b7581a5d8fa8721a1d7479a0dca6c08386328e19882/editdistance-0.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:09f01ed51746d90178af7dd7ea4ebb41497ef19f53c7f327e864421743dffb0a", size = 79853, upload-time = "2024-02-10T07:44:05.687Z" }, +] + +[[package]] +name = "evaluate" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "datasets" }, + { name = "dill" }, + { name = "fsspec", extra = ["http"] }, + { name = "huggingface-hub" }, + { name = "multiprocess" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "xxhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/d0/0c17a8e6e8dc7245f22dea860557c32bae50fc4d287ae030cb0e8ab8720f/evaluate-0.4.6.tar.gz", hash = "sha256:e07036ca12b3c24331f83ab787f21cc2dbf3631813a1631e63e40897c69a3f21", size = 65716, upload-time = "2025-09-18T13:06:30.581Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/af/3e990d8d4002bbc9342adb4facd59506e653da93b2417de0fa6027cb86b1/evaluate-0.4.6-py3-none-any.whl", hash = "sha256:bca85bc294f338377b7ac2f861e21c308b11b2a285f510d7d5394d5df437db29", size = 84069, upload-time = "2025-09-18T13:06:29.265Z" }, +] + +[[package]] +name = "executing" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/28/c14e053b6762b1044f34a13aab6859bbf40456d37d23aa286ac24cfd9a5d/executing-2.2.1.tar.gz", hash = "sha256:3632cc370565f6648cc328b32435bd120a1e4ebb20c77e3fdde9a13cd1e533c4", size = 1129488, upload-time = "2025-09-01T09:48:10.866Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, +] + +[[package]] +name = "filelock" +version = "3.32.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/57/3ba6e6cb097f85b855b00163d169f35365f44277df044dcf96d55b8f62a3/filelock-3.32.2.tar.gz", hash = "sha256:c33351e1f49cae33414acbc6d56784e6ecee82514ec90795da1161fc4836b5b8", size = 217172, upload-time = "2026-07-29T22:46:04.895Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/e8/72f8cef9fdfeffe06213fe8508039396ee48daa0e3259457ed766173bfd6/filelock-3.32.2-py3-none-any.whl", hash = "sha256:87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82", size = 98830, upload-time = "2026-07-29T22:46:03.52Z" }, +] + +[[package]] +name = "fonttools" +version = "4.63.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/84/69/c97f2c18e0db87d2c7b15da1974dace76ae938f1cfa22e2727a648b7ed43/fonttools-4.63.0.tar.gz", hash = "sha256:caeb583deeb5168e694b65cda8b4ee62abedfa66cf88488734466f2366b9c4e0", size = 3597189, upload-time = "2026-05-14T12:04:30.958Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/ef/b3c6b9b5be2f82416d73fe2ed2e96e2793cd80e7510bd6a17ca79cdd88ec/fonttools-4.63.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:37dd23e621e3b0aef1baa70a303b80aaf38449632cfc8fd2a55fb285bbccfc02", size = 2881131, upload-time = "2026-05-14T12:03:13.386Z" }, + { url = "https://files.pythonhosted.org/packages/44/a0/c815bea63117fa63e4e1c01f8a1110d2112fa003f838e6467094ec2432ce/fonttools-4.63.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a9faff9e0c1f76f9fd55899d2ce785832efebab37eb8ae13995853aef178bef0", size = 2426704, upload-time = "2026-05-14T12:03:15.801Z" }, + { url = "https://files.pythonhosted.org/packages/44/04/0b91d8e916e92ad1fac9e4624760baf0fd5ff2ead614c2f68fb21373f03f/fonttools-4.63.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef3048ef05dbb552b89817713d9cac912e00d0fde4a3105c00d29e52e10c89af", size = 5044298, upload-time = "2026-05-14T12:03:18.085Z" }, + { url = "https://files.pythonhosted.org/packages/77/c7/2342da9830e3e9d4870305ca5d2091d2a83284f2953079b7bdd3b5e029d8/fonttools-4.63.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:58dc6bb86a78d782f00f9190ca02c119cf5bbe2807536e361e18d42019f877d8", size = 4999800, upload-time = "2026-05-14T12:03:20.161Z" }, + { url = "https://files.pythonhosted.org/packages/e6/6d/67fe16c48d7ce050979b33f47e0d28a318f02da030602e944c34f7a16ef3/fonttools-4.63.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee08ebfa58f6e1aeff5697ab9582105bb620008c1caafb681e4c557e7483027b", size = 4982666, upload-time = "2026-05-14T12:03:22.87Z" }, + { url = "https://files.pythonhosted.org/packages/f2/00/3bbab338c07c71fa56269953845e92c951a61457bbbb0f1022551ea266d9/fonttools-4.63.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:27fdc65af8da6f88b9c6121c47a464cbe359fcfff7ff6fc2d37a1f395d755b78", size = 5133598, upload-time = "2026-05-14T12:03:25.168Z" }, + { url = "https://files.pythonhosted.org/packages/62/f2/aa27c7f98db5b064883dadcc5283947e81e034de42e22a33675878d98b54/fonttools-4.63.0-cp312-cp312-win32.whl", hash = "sha256:af2fd1664d00a397d75f806985ddb36282091c2131a73a6485c23b4a34722263", size = 2292575, upload-time = "2026-05-14T12:03:27.496Z" }, + { url = "https://files.pythonhosted.org/packages/87/36/cccb9bc2a6ab63d1b2980374f0dca72ce95ae267c9b4cfe77455bb70d0d4/fonttools-4.63.0-cp312-cp312-win_amd64.whl", hash = "sha256:59ac449f8cca9b4ffa08d2e7bbadad87ce710d69d1eda5c3c1ce579baa987272", size = 2343211, upload-time = "2026-05-14T12:03:30.057Z" }, + { url = "https://files.pythonhosted.org/packages/0f/8d/d8fec3dcde2963f8c908fb315e5ff2cd0ac34f82394bbbf73a2aa5145ce3/fonttools-4.63.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:cd7e9857e5e63738b9d9fd707bc1f59c8b09e5177726d23664db393c59bb08bd", size = 2876062, upload-time = "2026-05-14T12:03:32.554Z" }, + { url = "https://files.pythonhosted.org/packages/ef/71/d935dc54e4ff121bfdd11e08702db63a7e6f25af21d8a3d7b7212df53641/fonttools-4.63.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c2a2a42198b696a6f48fad91709afb55176e66a5e566131219dba372fb7f8c59", size = 2424594, upload-time = "2026-05-14T12:03:34.86Z" }, + { url = "https://files.pythonhosted.org/packages/8e/40/e76320afa1df918e146155ef239b1719ee266092e96f5423bfd075affba1/fonttools-4.63.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e874792a8212b44583ea02189d9e693906b2f78b261f372f95d6c563210ac1d", size = 5024840, upload-time = "2026-05-14T12:03:36.745Z" }, + { url = "https://files.pythonhosted.org/packages/ce/36/0b805d8c485f872f65a509cbe3b58a5d0d17bee855333b54a150c79d3061/fonttools-4.63.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:22135da48a348785c5e2d5d2d9d6bec5ed44adacbaeb9db12d9493bf6c6bfa68", size = 4975801, upload-time = "2026-05-14T12:03:38.833Z" }, + { url = "https://files.pythonhosted.org/packages/c8/26/2cee03d0aa083ab022da5c07aff9ed3f689da1defb81ad6917c9627896da/fonttools-4.63.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ccf41f2efdf56994d22d73bef4ced1052161958169428d06ba9724ea9e9a64be", size = 4965009, upload-time = "2026-05-14T12:03:41.494Z" }, + { url = "https://files.pythonhosted.org/packages/7e/48/cc4b66d9058c0d0982c833fad10127c4b0e9324606aafa41382295ca4102/fonttools-4.63.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9ced0bd02ac751dd6319b0da88aaef24414e3b0dbc32bb4f24944821a3741a27", size = 5105892, upload-time = "2026-05-14T12:03:43.525Z" }, + { url = "https://files.pythonhosted.org/packages/d8/1f/a98a30a814b9ddef3a2e706025f90b9e0bc94890e6cb15254bc86547d11a/fonttools-4.63.0-cp313-cp313-win32.whl", hash = "sha256:85be818f5506e8a7753153def2c9550178f0ecae6a47b5e0e8dbb23f7cc90380", size = 2291313, upload-time = "2026-05-14T12:03:45.594Z" }, + { url = "https://files.pythonhosted.org/packages/92/46/5177b01f3b4abfdd4409f31cca4ab279c9343a26efbe9ec78c97fc612e02/fonttools-4.63.0-cp313-cp313-win_amd64.whl", hash = "sha256:ba04cb5891d4c0c21b6da95eda8d7b090021508a294fff33464fc7d241e0856b", size = 2342299, upload-time = "2026-05-14T12:03:47.414Z" }, + { url = "https://files.pythonhosted.org/packages/27/d2/23d25e3f247b328be58d04a4c9f894178a0d1eda7d42867cfb388adaf416/fonttools-4.63.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fd1e3094f42d806d3d7c79162fc59e5910fcbe3a7360c385b8da969bc4493745", size = 2875338, upload-time = "2026-05-14T12:03:50.052Z" }, + { url = "https://files.pythonhosted.org/packages/cd/58/7dfa0c761cb3b2964e2a84c4dc986c926a87de0cb9fb60d5b28ded3f2914/fonttools-4.63.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6e528da43bc3791085f8cb6141b1d13e459226790240340fcbb4625649238b03", size = 2422661, upload-time = "2026-05-14T12:03:52.154Z" }, + { url = "https://files.pythonhosted.org/packages/dd/87/64cfa18a7a1621d17b7f4502b2b0ed8a135a90c3db51ea590ee99043e76b/fonttools-4.63.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b2248c5decb223562f7902ff6325077a073f608ee8e33e88ad88db734eb9f49", size = 5010526, upload-time = "2026-05-14T12:03:54.647Z" }, + { url = "https://files.pythonhosted.org/packages/36/e1/a8933a72c45a87177fbde2696e0d0755c8c9062f8c077a961c6215fa27b1/fonttools-4.63.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:308f957cdeaf8abe4e5f2f124902ef405448af92c90f80e302a3b771c2e6116b", size = 4923946, upload-time = "2026-05-14T12:03:56.984Z" }, + { url = "https://files.pythonhosted.org/packages/27/60/872e6e233b8c5e8b41413796ff18b7fe479661bd40147e071b450dfad7a1/fonttools-4.63.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bf00f21eb5fb721dbaf73d1e9da6d02a1af7768f2ebcf9798be98beab8ba90f6", size = 4962489, upload-time = "2026-05-14T12:03:59.443Z" }, + { url = "https://files.pythonhosted.org/packages/30/c4/83c24f2ec38b90cfda84bf4b1a1f49df80e84a1db4e7ac6e0d41bf23bc39/fonttools-4.63.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c1aaa4b9c75798400ac043ce04d74e7830376c85095a5a6ed7cba2f17a266bf4", size = 5071870, upload-time = "2026-05-14T12:04:02.122Z" }, + { url = "https://files.pythonhosted.org/packages/de/40/3ae22b60ff1d41ce0bd044b31238cdc72cef99f28b976f1e128ebd618c9b/fonttools-4.63.0-cp314-cp314-win32.whl", hash = "sha256:22693918177bd9ceabec4736d338045f357769416fc6b0b2508eefef75b08616", size = 2295026, upload-time = "2026-05-14T12:04:04.47Z" }, + { url = "https://files.pythonhosted.org/packages/c3/d4/98078064ccc76b45cb0f6c002452011e93c4bd26f6850344f0951cc1fe89/fonttools-4.63.0-cp314-cp314-win_amd64.whl", hash = "sha256:7d782fac32985914c351556f68ac0855391572bcd87de50e05970d3cd4c96fc5", size = 2347454, upload-time = "2026-05-14T12:04:06.752Z" }, + { url = "https://files.pythonhosted.org/packages/49/4e/652d1580c5f4e39f7d103b0c793e4773129ad633dce4addd0cf4dfebde02/fonttools-4.63.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6db5140a60a5d731d21ec076745b40a310607731b0a565b50776393188649001", size = 2958152, upload-time = "2026-05-14T12:04:08.706Z" }, + { url = "https://files.pythonhosted.org/packages/0e/55/ad864c9a9b219f552eb46b32cd7906c466e5a578ba0c3abfcc0fe7413eb6/fonttools-4.63.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7d76edbff9014094dbf03bd2d074709dfa6ec7aba13d838c937a2b33d2d6a86e", size = 2460809, upload-time = "2026-05-14T12:04:10.783Z" }, + { url = "https://files.pythonhosted.org/packages/ea/2b/0aa8db70f18cf52e49b4ed5ecec68547f981160bf5ded3b5aed6faa0a6f9/fonttools-4.63.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0eac00b9118c3c2f87d272e45341871c5b3066baa3c86897fa634a7c3fb59096", size = 5148649, upload-time = "2026-05-14T12:04:12.747Z" }, + { url = "https://files.pythonhosted.org/packages/7f/63/18e4369c25043096f1048e0c9915951adc4f842bd81c6b18155824d6fa99/fonttools-4.63.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51394295f1a51de8b5f30bdb1e1b9a4231536c7064ef5c6e211eec19fa36036f", size = 4932147, upload-time = "2026-05-14T12:04:14.806Z" }, + { url = "https://files.pythonhosted.org/packages/a1/3f/67f3eac2ffd8a98446c5022f8ed3864eac878a5ff7af8df4c8286dba16cc/fonttools-4.63.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9e12f105d2b6342c559c298afb674006bb2893afc7102dcf8a1b55b0486b4e40", size = 5027237, upload-time = "2026-05-14T12:04:17.675Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ba/4e6214cb38a7b04779e97bb7636de9a5c7f20af7018d03dee0b64c08510a/fonttools-4.63.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:796f27556dbe094c4824f75ca85267e4df776c79036c8441469a4df37038c196", size = 5053933, upload-time = "2026-05-14T12:04:20.818Z" }, + { url = "https://files.pythonhosted.org/packages/34/3b/214dcc19ee31d3d38fb5ad2755c11ef0514e5dc300bbaf41c0b69f393799/fonttools-4.63.0-cp314-cp314t-win32.whl", hash = "sha256:948428a275741f0b64b113c955425a953314f4b9ab9997f73a72c83e68e569c8", size = 2359326, upload-time = "2026-05-14T12:04:24.22Z" }, + { url = "https://files.pythonhosted.org/packages/dd/1e/3ff1a9b523058c2eeb6a9d50f5574e2a738200d0d94107d5bc4105e8da3f/fonttools-4.63.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6d4741eb179121cab9eea4cb2393d24492373a260d7945006358c08cfbf45419", size = 2425829, upload-time = "2026-05-14T12:04:26.829Z" }, + { url = "https://files.pythonhosted.org/packages/2c/47/c99d5268f354002ce80f8d029cd9d7d872969da1de8b93d32de4dc56d6f4/fonttools-4.63.0-py3-none-any.whl", hash = "sha256:445af2eab030a16b9171ea8bdda7ebf7d96bda2df88ee182a464252f6e05e20d", size = 1164562, upload-time = "2026-05-14T12:04:29.092Z" }, +] + +[[package]] +name = "frozenlist" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, + { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, + { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, + { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, + { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, + { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, + { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, + { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, + { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, + { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, + { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, + { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, + { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, + { url = "https://files.pythonhosted.org/packages/2d/40/0832c31a37d60f60ed79e9dfb5a92e1e2af4f40a16a29abcc7992af9edff/frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a", size = 85717, upload-time = "2025-10-06T05:36:27.341Z" }, + { url = "https://files.pythonhosted.org/packages/30/ba/b0b3de23f40bc55a7057bd38434e25c34fa48e17f20ee273bbde5e0650f3/frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7", size = 49651, upload-time = "2025-10-06T05:36:28.855Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ab/6e5080ee374f875296c4243c381bbdef97a9ac39c6e3ce1d5f7d42cb78d6/frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40", size = 49417, upload-time = "2025-10-06T05:36:29.877Z" }, + { url = "https://files.pythonhosted.org/packages/d5/4e/e4691508f9477ce67da2015d8c00acd751e6287739123113a9fca6f1604e/frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027", size = 234391, upload-time = "2025-10-06T05:36:31.301Z" }, + { url = "https://files.pythonhosted.org/packages/40/76/c202df58e3acdf12969a7895fd6f3bc016c642e6726aa63bd3025e0fc71c/frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822", size = 233048, upload-time = "2025-10-06T05:36:32.531Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c0/8746afb90f17b73ca5979c7a3958116e105ff796e718575175319b5bb4ce/frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121", size = 226549, upload-time = "2025-10-06T05:36:33.706Z" }, + { url = "https://files.pythonhosted.org/packages/7e/eb/4c7eefc718ff72f9b6c4893291abaae5fbc0c82226a32dcd8ef4f7a5dbef/frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5", size = 239833, upload-time = "2025-10-06T05:36:34.947Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/e5c02187cf704224f8b21bee886f3d713ca379535f16893233b9d672ea71/frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e", size = 245363, upload-time = "2025-10-06T05:36:36.534Z" }, + { url = "https://files.pythonhosted.org/packages/1f/96/cb85ec608464472e82ad37a17f844889c36100eed57bea094518bf270692/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11", size = 229314, upload-time = "2025-10-06T05:36:38.582Z" }, + { url = "https://files.pythonhosted.org/packages/5d/6f/4ae69c550e4cee66b57887daeebe006fe985917c01d0fff9caab9883f6d0/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1", size = 243365, upload-time = "2025-10-06T05:36:40.152Z" }, + { url = "https://files.pythonhosted.org/packages/7a/58/afd56de246cf11780a40a2c28dc7cbabbf06337cc8ddb1c780a2d97e88d8/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1", size = 237763, upload-time = "2025-10-06T05:36:41.355Z" }, + { url = "https://files.pythonhosted.org/packages/cb/36/cdfaf6ed42e2644740d4a10452d8e97fa1c062e2a8006e4b09f1b5fd7d63/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8", size = 240110, upload-time = "2025-10-06T05:36:42.716Z" }, + { url = "https://files.pythonhosted.org/packages/03/a8/9ea226fbefad669f11b52e864c55f0bd57d3c8d7eb07e9f2e9a0b39502e1/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed", size = 233717, upload-time = "2025-10-06T05:36:44.251Z" }, + { url = "https://files.pythonhosted.org/packages/1e/0b/1b5531611e83ba7d13ccc9988967ea1b51186af64c42b7a7af465dcc9568/frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496", size = 39628, upload-time = "2025-10-06T05:36:45.423Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cf/174c91dbc9cc49bc7b7aab74d8b734e974d1faa8f191c74af9b7e80848e6/frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231", size = 43882, upload-time = "2025-10-06T05:36:46.796Z" }, + { url = "https://files.pythonhosted.org/packages/c1/17/502cd212cbfa96eb1388614fe39a3fc9ab87dbbe042b66f97acb57474834/frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62", size = 39676, upload-time = "2025-10-06T05:36:47.8Z" }, + { url = "https://files.pythonhosted.org/packages/d2/5c/3bbfaa920dfab09e76946a5d2833a7cbdf7b9b4a91c714666ac4855b88b4/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94", size = 89235, upload-time = "2025-10-06T05:36:48.78Z" }, + { url = "https://files.pythonhosted.org/packages/d2/d6/f03961ef72166cec1687e84e8925838442b615bd0b8854b54923ce5b7b8a/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c", size = 50742, upload-time = "2025-10-06T05:36:49.837Z" }, + { url = "https://files.pythonhosted.org/packages/1e/bb/a6d12b7ba4c3337667d0e421f7181c82dda448ce4e7ad7ecd249a16fa806/frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52", size = 51725, upload-time = "2025-10-06T05:36:50.851Z" }, + { url = "https://files.pythonhosted.org/packages/bc/71/d1fed0ffe2c2ccd70b43714c6cab0f4188f09f8a67a7914a6b46ee30f274/frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51", size = 284533, upload-time = "2025-10-06T05:36:51.898Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1f/fb1685a7b009d89f9bf78a42d94461bc06581f6e718c39344754a5d9bada/frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65", size = 292506, upload-time = "2025-10-06T05:36:53.101Z" }, + { url = "https://files.pythonhosted.org/packages/e6/3b/b991fe1612703f7e0d05c0cf734c1b77aaf7c7d321df4572e8d36e7048c8/frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82", size = 274161, upload-time = "2025-10-06T05:36:54.309Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ec/c5c618767bcdf66e88945ec0157d7f6c4a1322f1473392319b7a2501ded7/frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714", size = 294676, upload-time = "2025-10-06T05:36:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ce/3934758637d8f8a88d11f0585d6495ef54b2044ed6ec84492a91fa3b27aa/frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d", size = 300638, upload-time = "2025-10-06T05:36:56.758Z" }, + { url = "https://files.pythonhosted.org/packages/fc/4f/a7e4d0d467298f42de4b41cbc7ddaf19d3cfeabaf9ff97c20c6c7ee409f9/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506", size = 283067, upload-time = "2025-10-06T05:36:57.965Z" }, + { url = "https://files.pythonhosted.org/packages/dc/48/c7b163063d55a83772b268e6d1affb960771b0e203b632cfe09522d67ea5/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51", size = 292101, upload-time = "2025-10-06T05:36:59.237Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d0/2366d3c4ecdc2fd391e0afa6e11500bfba0ea772764d631bbf82f0136c9d/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e", size = 289901, upload-time = "2025-10-06T05:37:00.811Z" }, + { url = "https://files.pythonhosted.org/packages/b8/94/daff920e82c1b70e3618a2ac39fbc01ae3e2ff6124e80739ce5d71c9b920/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0", size = 289395, upload-time = "2025-10-06T05:37:02.115Z" }, + { url = "https://files.pythonhosted.org/packages/e3/20/bba307ab4235a09fdcd3cc5508dbabd17c4634a1af4b96e0f69bfe551ebd/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41", size = 283659, upload-time = "2025-10-06T05:37:03.711Z" }, + { url = "https://files.pythonhosted.org/packages/fd/00/04ca1c3a7a124b6de4f8a9a17cc2fcad138b4608e7a3fc5877804b8715d7/frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b", size = 43492, upload-time = "2025-10-06T05:37:04.915Z" }, + { url = "https://files.pythonhosted.org/packages/59/5e/c69f733a86a94ab10f68e496dc6b7e8bc078ebb415281d5698313e3af3a1/frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888", size = 48034, upload-time = "2025-10-06T05:37:06.343Z" }, + { url = "https://files.pythonhosted.org/packages/16/6c/be9d79775d8abe79b05fa6d23da99ad6e7763a1d080fbae7290b286093fd/frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042", size = 41749, upload-time = "2025-10-06T05:37:07.431Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c8/85da824b7e7b9b6e7f7705b2ecaf9591ba6f79c1177f324c2735e41d36a2/frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0", size = 86127, upload-time = "2025-10-06T05:37:08.438Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f", size = 49698, upload-time = "2025-10-06T05:37:09.48Z" }, + { url = "https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c", size = 49749, upload-time = "2025-10-06T05:37:10.569Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2", size = 231298, upload-time = "2025-10-06T05:37:11.993Z" }, + { url = "https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8", size = 232015, upload-time = "2025-10-06T05:37:13.194Z" }, + { url = "https://files.pythonhosted.org/packages/dc/94/be719d2766c1138148564a3960fc2c06eb688da592bdc25adcf856101be7/frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686", size = 225038, upload-time = "2025-10-06T05:37:14.577Z" }, + { url = "https://files.pythonhosted.org/packages/e4/09/6712b6c5465f083f52f50cf74167b92d4ea2f50e46a9eea0523d658454ae/frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e", size = 240130, upload-time = "2025-10-06T05:37:15.781Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d4/cd065cdcf21550b54f3ce6a22e143ac9e4836ca42a0de1022da8498eac89/frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a", size = 242845, upload-time = "2025-10-06T05:37:17.037Z" }, + { url = "https://files.pythonhosted.org/packages/62/c3/f57a5c8c70cd1ead3d5d5f776f89d33110b1addae0ab010ad774d9a44fb9/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128", size = 229131, upload-time = "2025-10-06T05:37:18.221Z" }, + { url = "https://files.pythonhosted.org/packages/6c/52/232476fe9cb64f0742f3fde2b7d26c1dac18b6d62071c74d4ded55e0ef94/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f", size = 240542, upload-time = "2025-10-06T05:37:19.771Z" }, + { url = "https://files.pythonhosted.org/packages/5f/85/07bf3f5d0fb5414aee5f47d33c6f5c77bfe49aac680bfece33d4fdf6a246/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7", size = 237308, upload-time = "2025-10-06T05:37:20.969Z" }, + { url = "https://files.pythonhosted.org/packages/11/99/ae3a33d5befd41ac0ca2cc7fd3aa707c9c324de2e89db0e0f45db9a64c26/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30", size = 238210, upload-time = "2025-10-06T05:37:22.252Z" }, + { url = "https://files.pythonhosted.org/packages/b2/60/b1d2da22f4970e7a155f0adde9b1435712ece01b3cd45ba63702aea33938/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7", size = 231972, upload-time = "2025-10-06T05:37:23.5Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ab/945b2f32de889993b9c9133216c068b7fcf257d8595a0ac420ac8677cab0/frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806", size = 40536, upload-time = "2025-10-06T05:37:25.581Z" }, + { url = "https://files.pythonhosted.org/packages/59/ad/9caa9b9c836d9ad6f067157a531ac48b7d36499f5036d4141ce78c230b1b/frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0", size = 44330, upload-time = "2025-10-06T05:37:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/82/13/e6950121764f2676f43534c555249f57030150260aee9dcf7d64efda11dd/frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b", size = 40627, upload-time = "2025-10-06T05:37:28.075Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c7/43200656ecc4e02d3f8bc248df68256cd9572b3f0017f0a0c4e93440ae23/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d", size = 89238, upload-time = "2025-10-06T05:37:29.373Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/55c5f0689b9c0fb765055629f472c0de484dcaf0acee2f7707266ae3583c/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed", size = 50738, upload-time = "2025-10-06T05:37:30.792Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7d/b7282a445956506fa11da8c2db7d276adcbf2b17d8bb8407a47685263f90/frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930", size = 51739, upload-time = "2025-10-06T05:37:32.127Z" }, + { url = "https://files.pythonhosted.org/packages/62/1c/3d8622e60d0b767a5510d1d3cf21065b9db874696a51ea6d7a43180a259c/frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c", size = 284186, upload-time = "2025-10-06T05:37:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/2d/14/aa36d5f85a89679a85a1d44cd7a6657e0b1c75f61e7cad987b203d2daca8/frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24", size = 292196, upload-time = "2025-10-06T05:37:36.107Z" }, + { url = "https://files.pythonhosted.org/packages/05/23/6bde59eb55abd407d34f77d39a5126fb7b4f109a3f611d3929f14b700c66/frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37", size = 273830, upload-time = "2025-10-06T05:37:37.663Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3f/22cff331bfad7a8afa616289000ba793347fcd7bc275f3b28ecea2a27909/frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a", size = 294289, upload-time = "2025-10-06T05:37:39.261Z" }, + { url = "https://files.pythonhosted.org/packages/a4/89/5b057c799de4838b6c69aa82b79705f2027615e01be996d2486a69ca99c4/frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2", size = 300318, upload-time = "2025-10-06T05:37:43.213Z" }, + { url = "https://files.pythonhosted.org/packages/30/de/2c22ab3eb2a8af6d69dc799e48455813bab3690c760de58e1bf43b36da3e/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef", size = 282814, upload-time = "2025-10-06T05:37:45.337Z" }, + { url = "https://files.pythonhosted.org/packages/59/f7/970141a6a8dbd7f556d94977858cfb36fa9b66e0892c6dd780d2219d8cd8/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe", size = 291762, upload-time = "2025-10-06T05:37:46.657Z" }, + { url = "https://files.pythonhosted.org/packages/c1/15/ca1adae83a719f82df9116d66f5bb28bb95557b3951903d39135620ef157/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8", size = 289470, upload-time = "2025-10-06T05:37:47.946Z" }, + { url = "https://files.pythonhosted.org/packages/ac/83/dca6dc53bf657d371fbc88ddeb21b79891e747189c5de990b9dfff2ccba1/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a", size = 289042, upload-time = "2025-10-06T05:37:49.499Z" }, + { url = "https://files.pythonhosted.org/packages/96/52/abddd34ca99be142f354398700536c5bd315880ed0a213812bc491cff5e4/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e", size = 283148, upload-time = "2025-10-06T05:37:50.745Z" }, + { url = "https://files.pythonhosted.org/packages/af/d3/76bd4ed4317e7119c2b7f57c3f6934aba26d277acc6309f873341640e21f/frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df", size = 44676, upload-time = "2025-10-06T05:37:52.222Z" }, + { url = "https://files.pythonhosted.org/packages/89/76/c615883b7b521ead2944bb3480398cbb07e12b7b4e4d073d3752eb721558/frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd", size = 49451, upload-time = "2025-10-06T05:37:53.425Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a3/5982da14e113d07b325230f95060e2169f5311b1017ea8af2a29b374c289/frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79", size = 42507, upload-time = "2025-10-06T05:37:54.513Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, +] + +[[package]] +name = "fsspec" +version = "2026.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/10/a1/ae4e3e5003468d6391d2c77b6fa1cd73bd5d13511d81c642d7b28ac90ed4/fsspec-2026.6.0.tar.gz", hash = "sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a", size = 313646, upload-time = "2026-06-16T01:57:28.105Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", size = 203949, upload-time = "2026-06-16T01:57:26.358Z" }, +] + +[package.optional-dependencies] +http = [ + { name = "aiohttp" }, +] + +[[package]] +name = "ftfy" +version = "6.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a5/d3/8650919bc3c7c6e90ee3fa7fd618bf373cbbe55dff043bd67353dbb20cd8/ftfy-6.3.1.tar.gz", hash = "sha256:9b3c3d90f84fb267fe64d375a07b7f8912d817cf86009ae134aa03e1819506ec", size = 308927, upload-time = "2024-10-26T00:50:35.149Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/6e/81d47999aebc1b155f81eca4477a616a70f238a2549848c38983f3c22a82/ftfy-6.3.1-py3-none-any.whl", hash = "sha256:7c70eb532015cd2f9adb53f101fb6c7945988d023a085d127d1573dc49dd0083", size = 44821, upload-time = "2024-10-26T00:50:33.425Z" }, +] + +[[package]] +name = "func-timeout" +version = "4.3.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/0d/bf0567477f7281d9a3926c582bfef21bff7498fc0ffd3e9de21811896a0b/func_timeout-4.3.5.tar.gz", hash = "sha256:74cd3c428ec94f4edfba81f9b2f14904846d5ffccc27c92433b8b5939b5575dd", size = 44264, upload-time = "2019-08-19T21:32:07.43Z" } + +[[package]] +name = "fuzzysearch" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/b4/d92b429ba4bfe5461faa358f84092eceb5ea8985cd2e448daf38bba17737/fuzzysearch-0.8.1.tar.gz", hash = "sha256:e5f50962c6b1c3dfc6c8cdfd5e2604838c95cb10118e4cab518e5292e9e0b1c6", size = 39510, upload-time = "2025-11-11T08:47:10.321Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/5f/0b0b2a56d59472becda3b7788e532e25d6a74451679d7d99b411a24553e7/fuzzysearch-0.8.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d3caa5c635ac87c67f58b8a50792073014989e0689263c494e81f5d2ced24101", size = 31830, upload-time = "2025-11-11T08:46:42.163Z" }, + { url = "https://files.pythonhosted.org/packages/2b/ce/f75349a22df809ea7a4433ff9e24edfbc3cfa160600f159b50e0e4e272a4/fuzzysearch-0.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a8187ebcfea9f76387b57a618480ff536ff2a5da67d5dd31af4c252bf8842d45", size = 32507, upload-time = "2025-11-11T08:46:43.315Z" }, + { url = "https://files.pythonhosted.org/packages/36/3c/d193b6b80d88d7e3b2c3a45fd582dedcb3b47c3bf2ffb2b85083e3b3ba8a/fuzzysearch-0.8.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5d19cf0ebfefe54737a0477defc218123a0addb9d52f43711286aab94603e288", size = 55361, upload-time = "2025-11-11T08:46:44.714Z" }, + { url = "https://files.pythonhosted.org/packages/7d/36/71a9bd65e361083ed3cbab135cd3a82ae2b089cbca6608af6106acc9017c/fuzzysearch-0.8.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85e5a4f8099517b2bfa384668160fd06b020106e86c6077d21e93a6b4fb348eb", size = 57213, upload-time = "2025-11-11T08:46:45.826Z" }, + { url = "https://files.pythonhosted.org/packages/0f/7d/23f8d9d137187f5dfececd14f16f79e8257eb0fd2e85397306474f987710/fuzzysearch-0.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:ea17a28427fb750bbec47f2169f223f81a8388e81535f90d0590efd94489f803", size = 56287, upload-time = "2025-11-11T08:46:46.924Z" }, + { url = "https://files.pythonhosted.org/packages/ac/a6/c73e8ebc7e6e4ee7f8d2c509c232781dfc79451ebda2254915e0cef17566/fuzzysearch-0.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f4f11634b0d22dbd36513dae77b2f794d63b0275879da050b379f91bab398225", size = 56718, upload-time = "2025-11-11T08:46:47.921Z" }, + { url = "https://files.pythonhosted.org/packages/9b/3f/29d666a60f711a77354a5b23082fe210b4d0277e5d6e9e5146cf56162315/fuzzysearch-0.8.1-cp312-cp312-win32.whl", hash = "sha256:040693ed5c6c3b15807a240b7acf535cacd12771ca5094cec1df62c960010642", size = 35104, upload-time = "2025-11-11T08:46:49.378Z" }, + { url = "https://files.pythonhosted.org/packages/58/10/f2cef42d66311554723cfc1f7295ba1a83b3dd763845219bba3120bb3c08/fuzzysearch-0.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:46bb36a6c5a966d951125136379a8674377ceea9f755fd542536065a6d47d23e", size = 36766, upload-time = "2025-11-11T08:46:50.362Z" }, +] + +[[package]] +name = "gdown" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "filelock" }, + { name = "requests", extra = ["socks"] }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/b5/a45f62f20664031bf74a6aeb6f8d8cd5910e411bf90d756bd6b09bdc6c35/gdown-6.1.0.tar.gz", hash = "sha256:361c6e04c6ca335df50b9d71f40bcfe9ab70fb26a1b0e890a427267781389553", size = 269670, upload-time = "2026-05-30T11:56:21.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/56/a99f0f159cce5b26d267317d436afee184f45fc7911938757d7cbbd2d10c/gdown-6.1.0-py3-none-any.whl", hash = "sha256:38a36a94275b8272f684db469bbd73b4d1f64cbbc1751bcb993a1b2be8f013c8", size = 19216, upload-time = "2026-05-30T11:56:20.016Z" }, +] + +[[package]] +name = "google-api-core" +version = "2.33.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "googleapis-common-protos" }, + { name = "proto-plus" }, + { name = "protobuf" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/87/62/8fb1fb647d2788c950d69d6a769cd9d55c918ac1fc57be2f90b7e4029787/google_api_core-2.33.0.tar.gz", hash = "sha256:3a36bcc3e319783f4c97da41f6f45ea6ffcaa55848e341de16e09cb70243c2bb", size = 181607, upload-time = "2026-07-22T16:28:28.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/31/5056a347bb934ea04583c8b27916ef1501729c72638629545bce26ff4223/google_api_core-2.33.0-py3-none-any.whl", hash = "sha256:a2e22a0c1d0f03eafff1858b38cf46f832d5902b0c052235bf0ab8402929fbdc", size = 176462, upload-time = "2026-07-22T16:28:22.447Z" }, +] + +[[package]] +name = "google-auth" +version = "2.56.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/33/dbc946a407401b975f0719658f18e664ece2109f79ffd1ff3bf226c205f4/google_auth-2.56.2.tar.gz", hash = "sha256:e28f103ca8091fb7012b99c44243d7366c29863713b8e34a220c3322b7a07051", size = 365820, upload-time = "2026-07-21T21:53:28.188Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/63/50636aae68c9bf17c891c7eb18b49baa9bd6b31d2a97b8de4813a9fc8d1c/google_auth-2.56.2-py3-none-any.whl", hash = "sha256:c8270ea95b2697b74e3d8438ae9c5b898e38b623b915c7b5c5635921e7de68a6", size = 258588, upload-time = "2026-07-21T21:53:26.399Z" }, +] + +[package.optional-dependencies] +requests = [ + { name = "requests" }, +] + +[[package]] +name = "google-cloud-core" +version = "2.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/dd/1eef226e470369b26824a505c34482c0b493bc35fe8e0c6b003b5feca21a/google_cloud_core-2.6.0.tar.gz", hash = "sha256:e76149739f90fac1fc6757c09f47eaccb3145b54adbd7759b0f7c4b235f46c83", size = 36001, upload-time = "2026-05-07T08:04:04.124Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/4a/98da8930ab109c73d9a5d13782a9ebb81ea8c111f6d534a567b71d23e52b/google_cloud_core-2.6.0-py3-none-any.whl", hash = "sha256:6d63ac8e5eca6d9e4319d0a1e2265fadcd7f1049904378caecfa01cf52dd869e", size = 29390, upload-time = "2026-05-07T08:02:34.672Z" }, +] + +[[package]] +name = "google-cloud-storage" +version = "3.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, + { name = "google-cloud-core" }, + { name = "google-crc32c" }, + { name = "google-resumable-media" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e3/25/355ed97c1723c787dfaa888808d55db18371f82c38ff862357b1e902cd19/google_cloud_storage-3.13.0.tar.gz", hash = "sha256:d11d8706ea1520fba0f21043bcb7897caf7015d76ce1ad9a4f60237e4d7a9f6c", size = 17340960, upload-time = "2026-07-13T19:10:07.524Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/e8/b3678a0931ee7d4b3fdaf0813e6206d66e0922b2c26d912f308728b5b95a/google_cloud_storage-3.13.0-py3-none-any.whl", hash = "sha256:648af3ef8a6acc674e1359d3c920c67eb89a7a5ab66b336bd3ac43fed6b5ab84", size = 341428, upload-time = "2026-07-13T19:09:52.39Z" }, +] + +[[package]] +name = "google-crc32c" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113", size = 31300, upload-time = "2025-12-16T00:21:56.723Z" }, + { url = "https://files.pythonhosted.org/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb", size = 30867, upload-time = "2025-12-16T00:38:31.302Z" }, + { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, + { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, + { url = "https://files.pythonhosted.org/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962", size = 34437, upload-time = "2025-12-16T00:35:21.395Z" }, + { url = "https://files.pythonhosted.org/packages/d1/db/000f15b41724589b0e7bc24bc7a8967898d8d3bc8caf64c513d91ef1f6c0/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b", size = 31297, upload-time = "2025-12-16T00:23:20.709Z" }, + { url = "https://files.pythonhosted.org/packages/d7/0d/8ebed0c39c53a7e838e2a486da8abb0e52de135f1b376ae2f0b160eb4c1a/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27", size = 30867, upload-time = "2025-12-16T00:43:14.628Z" }, + { url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" }, + { url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" }, + { url = "https://files.pythonhosted.org/packages/92/b1/d3cbd4d988afb3d8e4db94ca953df429ed6db7282ed0e700d25e6c7bfc8d/google_crc32c-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f", size = 34435, upload-time = "2025-12-16T00:35:22.107Z" }, + { url = "https://files.pythonhosted.org/packages/21/88/8ecf3c2b864a490b9e7010c84fd203ec8cf3b280651106a3a74dd1b0ca72/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697", size = 31301, upload-time = "2025-12-16T00:24:48.527Z" }, + { url = "https://files.pythonhosted.org/packages/36/c6/f7ff6c11f5ca215d9f43d3629163727a272eabc356e5c9b2853df2bfe965/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651", size = 30868, upload-time = "2025-12-16T00:48:12.163Z" }, + { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" }, + { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, + { url = "https://files.pythonhosted.org/packages/08/a5/7b059810934a09fb3ccb657e0843813c1fee1183d3bc2c8041800374aa2c/google_crc32c-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2", size = 34878, upload-time = "2025-12-16T00:35:23.142Z" }, +] + +[[package]] +name = "google-genai" +version = "2.18.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "google-auth", extra = ["requests"] }, + { name = "httpx" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "sniffio" }, + { name = "tenacity" }, + { name = "typing-extensions" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/e3/1dd592243a0dfc3487ddfff7995f12686d0557ec953173dd9bdbeba2cb96/google_genai-2.18.1.tar.gz", hash = "sha256:a1e2be75c16234adc6641afd1ad4dd44218c9eec005d938bdc428585a048918a", size = 659694, upload-time = "2026-08-13T22:13:50.226Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/7d/5310f7a1cf290a6cb22eab37059610bf338ab7595892902591d2220cf825/google_genai-2.18.1-py3-none-any.whl", hash = "sha256:36a5949233e64a60f6cc4521bff7a76b7c569d0aa227bbe9fa642213b8a3a3b2", size = 1051129, upload-time = "2026-08-13T22:13:48.083Z" }, +] + +[[package]] +name = "google-resumable-media" +version = "2.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-crc32c" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/f8/1ca5781d6be9cb9f73f7d40f4958c4bd1226a60598e3e39e1d6aaf838c4b/google_resumable_media-2.10.0.tar.gz", hash = "sha256:e324bc9d0fdae4c52a08ae90456edc4e71ece858399e1217ac0eb3a51d6bc6ee", size = 2164570, upload-time = "2026-06-03T16:14:26.103Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/d8/00c6854ac1512bb9eaf13bd3f8f28222f7674947fc510a4ff7616f2efc80/google_resumable_media-2.10.0-py3-none-any.whl", hash = "sha256:88152884bee37b2bf36a0ab81ad8c7fd12212c9803dd981d77c1b35b02d34e7c", size = 81533, upload-time = "2026-06-03T16:13:12.51Z" }, +] + +[[package]] +name = "googleapis-common-protos" +version = "1.75.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/c8/f439cffde755cffa462bfbb156278fa6f9d09119719af9814b858fd4f81f/googleapis_common_protos-1.75.0.tar.gz", hash = "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd", size = 151035, upload-time = "2026-05-07T08:04:49.423Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631, upload-time = "2026-05-07T08:03:30.345Z" }, +] + +[[package]] +name = "greenlet" +version = "3.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/74/b13368064b09053253555d3f2839cc2684d22d5aed0d2ccffbf7a6736558/greenlet-3.5.4.tar.gz", hash = "sha256:0232ae1de90a8e07867bb127d7a6ba2301e859145489f25cda8a6096dabe1d20", size = 206538, upload-time = "2026-07-22T12:47:14.468Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/04/81bd731d6d1e3a469d9a4c36f5eb069bcf0cbb2d5d342c9fec22245b91fc/greenlet-3.5.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3d66250e8b09f182ede05490998c818b5961f7a3640332d44c4927caec7bbfe4", size = 295909, upload-time = "2026-07-22T11:38:09.261Z" }, + { url = "https://files.pythonhosted.org/packages/cc/dd/f5f22903a6ae70f5ea328ed0beaec92ad903f0e3b7d2845133b354abc4b8/greenlet-3.5.4-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c90e930c9c192e5b3ee9fb8bcd920ea3926155e2e3ded39fc697323addecee17", size = 612011, upload-time = "2026-07-22T12:26:40.69Z" }, + { url = "https://files.pythonhosted.org/packages/8e/10/92a4a88d12b915d74ea5b6d288e4afefda4771647caa34442c156f7a454f/greenlet-3.5.4-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:791fdfeeb9c6e0c7b10fa151bf110d2a6974866f13dcb5b1c7efae698245893a", size = 624299, upload-time = "2026-07-22T12:29:02.089Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f9/03e26be3487c5238e81f2b84714959a86ea8515a869828cf41f4fc54b34e/greenlet-3.5.4-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b7c895310363f310361e0fe2072af85269d2a2a285cd04c0c59e79a5e3670dcf", size = 629603, upload-time = "2026-07-22T12:43:43.456Z" }, + { url = "https://files.pythonhosted.org/packages/50/6d/0b14bb9db2989f32cd9fe7f76afedea01ee8bee3f87c07e69f24adfe7e63/greenlet-3.5.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f88193799d43dbf8c8a806d6405c9c52fe2af40bf75072a606357b33cc336c7f", size = 621541, upload-time = "2026-07-22T11:51:09.464Z" }, + { url = "https://files.pythonhosted.org/packages/57/6b/7c55ca72ef80d57c16c4a55210f82582622462dc4485799a30f4ec6f3372/greenlet-3.5.4-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:13b980043cb1b3134e81ea469da1250ddcc6bfe6d245bbaa59168d9cdc8f228f", size = 432554, upload-time = "2026-07-22T12:39:51.379Z" }, + { url = "https://files.pythonhosted.org/packages/48/3d/25e9a2d9eb6b2e8b7ca4e80a3a26cb887cce6c8e0a87c921164f11bc5574/greenlet-3.5.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b7a5f095767c4493afcd06067f2bb3b8716e3f3f9e92b99c88e7e99f885b3d4d", size = 1581444, upload-time = "2026-07-22T12:25:03.818Z" }, + { url = "https://files.pythonhosted.org/packages/b9/96/4c9bf2e2c408dcc0556edce69efa9f802e82223573c53240136a086821f1/greenlet-3.5.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:42afdc1ab5f66da8c586c32af9224a74a706b4f0ea0dc3a4188a0860a09c65c9", size = 1645842, upload-time = "2026-07-22T11:51:12.295Z" }, + { url = "https://files.pythonhosted.org/packages/b5/41/303ecb26a3a56122c0f4d4073ee078881847bd6b6f463ae0ec57ec20223b/greenlet-3.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:60149df8f462d1b230038e6590c23c3b4768bb5d6c022b3b6e82532b34b0b8a3", size = 247169, upload-time = "2026-07-22T11:38:19.893Z" }, + { url = "https://files.pythonhosted.org/packages/a4/e3/ef56864b4c35fcb3eb3b41b869f6cc46f4cd3f5e2c68e74acde8ac433951/greenlet-3.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:77d6ce04fed0d9aeed42e0f37923cc43eba9b027bdd9c34546bb4ccd143d0fe0", size = 245565, upload-time = "2026-07-22T11:38:27.061Z" }, + { url = "https://files.pythonhosted.org/packages/c0/9a/e51225dcd58713f16ccbdcc501a8da21098ea14515b7870f1f94459e5ff5/greenlet-3.5.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:24e61b88cb7e1b1d794b32a10cc346ac779681d6d74ff137a3e0a444d2bf1f02", size = 294831, upload-time = "2026-07-22T11:38:53.389Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ea/de50a50fadf979713ab18b46f22ad5ff5f2dcfc637a3ebdecf669801e1a5/greenlet-3.5.4-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:870d730fec833f5a06906a32596cc099b9161594642a92a520b7a88911c95356", size = 614619, upload-time = "2026-07-22T12:26:42.282Z" }, + { url = "https://files.pythonhosted.org/packages/db/c7/2aae27fea41205b8650294c301f042a2a4bb6155eea48c995b890a92f2c1/greenlet-3.5.4-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ec5ff0d1878df6af3bf9b638a5a92a7d5693291de77c91bff10fa48519c604ef", size = 627021, upload-time = "2026-07-22T12:29:03.445Z" }, + { url = "https://files.pythonhosted.org/packages/1b/80/fb4d4788bbc8e54761f1fc88533af9523a6e86299fa113d6e8a8503ed9fc/greenlet-3.5.4-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:07bd44616608d873d06735b63ef1a88191d6ca57c8d291d6559c71bc14c0893c", size = 632845, upload-time = "2026-07-22T12:43:45.19Z" }, + { url = "https://files.pythonhosted.org/packages/eb/56/79fd826f9ccaae0b84e1b4ef68dabba5e105bb044ffcd448a0b782fcba9a/greenlet-3.5.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d84d993f6e575c950d91a23c1345d18fe1a4310d447bf630849d7809196b52f0", size = 624002, upload-time = "2026-07-22T11:51:11.391Z" }, + { url = "https://files.pythonhosted.org/packages/42/e3/6086fa578ebb72772722cdc4bcd628459814b42e0c2db1e3cbd6552b3271/greenlet-3.5.4-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:3529a8a933582ad19e224792cac7372489526576b75b4c124e8e4f29948f4861", size = 435053, upload-time = "2026-07-22T12:39:52.715Z" }, + { url = "https://files.pythonhosted.org/packages/0a/1a/27319f97e731298513dcba1a2e91b63e9d8811d9de22130f960b129b1bf1/greenlet-3.5.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:58023945f421093de5e6fa108c0985a8659d43f49e0216da25099369a121bcbd", size = 1581533, upload-time = "2026-07-22T12:25:05.322Z" }, + { url = "https://files.pythonhosted.org/packages/b1/6d/24240bf562e9786dd2799ee0a4a4dadb4ded22510f41b20245099159ac8c/greenlet-3.5.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bae2728e1897aa8df8cb1af38cd48b3a743aefe29372de7b8b7a9f532501e69f", size = 1645781, upload-time = "2026-07-22T11:51:14.805Z" }, + { url = "https://files.pythonhosted.org/packages/c1/5a/442ab1a9ef7ca6bf7210e5397a95972206a91a31033a03c8900866a10039/greenlet-3.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:ca5726c0b08ca35ae873557266a78b2c3f3b2b7d7401aa5ff886c2045dd0111c", size = 247133, upload-time = "2026-07-22T11:39:20.661Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e6/9160210222386b1a378ff94db846b9508ca24a121cf684991561fdb69280/greenlet-3.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:7c1303791d603080cac6fc3b34df51c3b75b723739c282c8029e48a0d241672f", size = 245500, upload-time = "2026-07-22T11:40:22.185Z" }, + { url = "https://files.pythonhosted.org/packages/a5/a7/6ab1d4f9cd548d15ab90da29947f2076100130bb179b0bde59f795a459e3/greenlet-3.5.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:7e8afa5eac028f8140ceafe5ceec66e6aa127ddcb21452d2a564dcd2900b5f22", size = 295410, upload-time = "2026-07-22T11:40:35.747Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7a/422f63b4715cbc0b24385305407adf38b48f6bb68b3e6b04090e994d0f5a/greenlet-3.5.4-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73b37afe369021423ea53dd3123e04bffa7e93ac64429b9f50835b2e4fcae7cf", size = 661286, upload-time = "2026-07-22T12:26:43.8Z" }, + { url = "https://files.pythonhosted.org/packages/d0/31/5a1cac663bf5582190c5a714ef81364f03cde232227f39748f8ae4c11da5/greenlet-3.5.4-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3ef964f56dfcb6f9bbef2a190d9126795eac408716aeae47b5e7c73c32aafca9", size = 673517, upload-time = "2026-07-22T12:29:04.815Z" }, + { url = "https://files.pythonhosted.org/packages/9c/bf/250c2921c7b585dde12f5239e313ca2dcbc464d161ecca36e4e6ef21762d/greenlet-3.5.4-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cef589bc65fae02d10bca2ac341191c5b33acc2967892ebf4fcbd10eabb7a74c", size = 677968, upload-time = "2026-07-22T12:43:46.788Z" }, + { url = "https://files.pythonhosted.org/packages/15/4a/2a82a1e3f8aaca020853ac8d12211280ca2b231aa08ea39f636f1060c319/greenlet-3.5.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c53ff01a5c53a40f2c16820ebc56d7c61a77f5fbe009dadd96292d5682f80f8", size = 670917, upload-time = "2026-07-22T11:51:13.589Z" }, + { url = "https://files.pythonhosted.org/packages/18/40/10bfcf6513558d82f7b95dd728001c63bd388259fe27d3e30ae01f103430/greenlet-3.5.4-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:dfc41ae893d9ceaf22c824f2153a88b30651b20e8758c2cd9ac143f23640563c", size = 480643, upload-time = "2026-07-22T12:39:54.149Z" }, + { url = "https://files.pythonhosted.org/packages/68/b0/e379a152b17bfdfa95795af4049e37c0fd1b4d81f020d426db104ed07c77/greenlet-3.5.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ecca4d80d55a01ad6b23b33262662956149fbb7b2c6be2910f1705921958cbf3", size = 1628478, upload-time = "2026-07-22T12:25:06.678Z" }, + { url = "https://files.pythonhosted.org/packages/5e/43/bffdfa64f7317f954c5c1230b5dd5922676ce198689a68c1ac1ed4b1b1a5/greenlet-3.5.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ffbc533e0eaf8e80d8471411646ab88fe58f641d508c0b02b24494479f4d9ec", size = 1692021, upload-time = "2026-07-22T11:51:17.008Z" }, + { url = "https://files.pythonhosted.org/packages/d0/11/f799f9637e2c6e9b0b716015e339040598b058cf7654dfc0d67468b177ed/greenlet-3.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:305f69e6c4523d7f6979ed001cff4e5853c063e5da04880296603aa0227e544c", size = 248031, upload-time = "2026-07-22T11:40:11.007Z" }, + { url = "https://files.pythonhosted.org/packages/05/75/625bcdd74d5e6b2dca1ecba3c3ac77bcf8a026c21a649a46cef23e421f97/greenlet-3.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:f260930bbbbcf9caee661211235a5111c86dfe5832fdf6ae4570da1e0995320f", size = 246892, upload-time = "2026-07-22T11:40:27.357Z" }, + { url = "https://files.pythonhosted.org/packages/ec/69/35c62ed49c320cb4d98e14698ccca5467d3bfe683984172be9cb564d9ce3/greenlet-3.5.4-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:41ddab54e4b238f4a6c323f39b4e59e176affd5a94d461a9fb7583dac74240a3", size = 305571, upload-time = "2026-07-22T11:40:31.659Z" }, + { url = "https://files.pythonhosted.org/packages/5b/6c/64d60216b3640dcb0b62d913dd9e0d80030c09115bb2e4ba70c95d10ca45/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b3dabe3e2809013052c68bdf0b7fa5f5f2859c43a80803131ad61af9cabd7867", size = 672568, upload-time = "2026-07-22T12:26:45.298Z" }, + { url = "https://files.pythonhosted.org/packages/5c/de/ba3ab0a96292e53039530333b0d2ae18d9e508f3a325cd7bf15f8172944c/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:27d3f00718634d4520a3a150154ac5da36f257869d41321953375b90bfbbc72c", size = 680076, upload-time = "2026-07-22T12:29:06.125Z" }, + { url = "https://files.pythonhosted.org/packages/ae/db/24a10af12bf8e639cec46c38b9ce1a282543ba42ff4fb0b31a970f1ab603/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:39169a11d87a6a263afda3e9a27d1df16d0f919d40a4837cc73986c9884c0dd8", size = 681690, upload-time = "2026-07-22T12:43:48.109Z" }, + { url = "https://files.pythonhosted.org/packages/a5/be/aeada79083c6f1c15f45d77a332f9c441af263ee298e3eb17522cd337d22/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbd60b5763c6543c1827e48faaf14ea9bfbad245f52b1a4d76a2a2d8884c6c66", size = 676733, upload-time = "2026-07-22T11:51:16.027Z" }, + { url = "https://files.pythonhosted.org/packages/f4/60/44a2eca7b9fd71ae0fae7ff184da1cd3169d176652b97aa1cffcbb0ef961/greenlet-3.5.4-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:bd3d1145f603b2db19feb9078c2e6855eb7c67e15580c010ed815cee519b86fd", size = 510263, upload-time = "2026-07-22T12:39:55.678Z" }, + { url = "https://files.pythonhosted.org/packages/e9/10/2392fc3a98948652ef5fd1e7275c04f861dd13f74b78a2b4309f4ee4d090/greenlet-3.5.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f00f910f0e7b35416c63b23ad78b769aeccfc1775f712b43c4ee525624a2eef7", size = 1637327, upload-time = "2026-07-22T12:25:07.879Z" }, + { url = "https://files.pythonhosted.org/packages/55/c6/e7237a3dfa1f205ed0d9ea1e46d70bd2811b32d516266399fb59d28ab90a/greenlet-3.5.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:91c26423753b92caf41ab3f98fd547d7374d4d9fc2d85be041886c1579d9255e", size = 1697493, upload-time = "2026-07-22T11:51:19.214Z" }, + { url = "https://files.pythonhosted.org/packages/55/e3/4ba8154ba2a3d43729e499f72471b4b5c993f3826d3e24da81d5f06d6572/greenlet-3.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:ee032b91fd8ec29ec6c4cea2b8c561b178435134bd0752c7334b94e9c736c132", size = 251637, upload-time = "2026-07-22T11:40:37.44Z" }, + { url = "https://files.pythonhosted.org/packages/90/03/e3f96dfc100261a29545ddc8270cafe58f9195b6651466910e820910de77/greenlet-3.5.4-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:178111881dd7a6c946471fda85485ec796e1043c2b939f694b096e2ecf986809", size = 296076, upload-time = "2026-07-22T11:39:38.364Z" }, + { url = "https://files.pythonhosted.org/packages/a4/3d/da52d208e5c977bce8667e784729e584e38b5785f4c1ec0f4c836e9a1c42/greenlet-3.5.4-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d92df08dd65fede97fc37aad36c2e9dcda3b31c467f8e0c2c096456cb818e927", size = 666870, upload-time = "2026-07-22T12:26:46.691Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8a/7e6dee25cb8a8cf9b362c8e597cc269593378bd916f16c736c059a52e85a/greenlet-3.5.4-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:99e8f8c4ebc4fd80aa26c1280ae9ad43a0976e786349703a181cf0bae60413e5", size = 677678, upload-time = "2026-07-22T12:29:07.508Z" }, + { url = "https://files.pythonhosted.org/packages/51/a7/dafc7415d430b0a43a16396eb49ecb3b62fd720877fb259cc4dcfaf5f31e/greenlet-3.5.4-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1f17e362d78e37559e0506c5a7d066bdd45073c36a0127a543e8a0df27242ff3", size = 681428, upload-time = "2026-07-22T12:43:49.623Z" }, + { url = "https://files.pythonhosted.org/packages/6c/21/5a38699fa45de749e3857d93b8f07e4c20489e77c2d35d915a2e1c456606/greenlet-3.5.4-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:394de08dad5ffcb1f50c2159d93e398d9d2da3ed437645eaa54771fa720db9f0", size = 676067, upload-time = "2026-07-22T11:51:18.163Z" }, + { url = "https://files.pythonhosted.org/packages/2e/d9/6298f3432de301d4718766cf934bd73c418c73f81fbb77247319364b0d96/greenlet-3.5.4-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:cd320d998cbaa032932830448e39abf3c6a12901295e386e8114db926e10cffb", size = 487446, upload-time = "2026-07-22T12:39:57.044Z" }, + { url = "https://files.pythonhosted.org/packages/5e/ba/863116ab8ff1ca7a729e327800268939d182db47aa433db70e216e7d9194/greenlet-3.5.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:c883d61f2282d72c767a14936641b3efcbde9d82f1080712aaea0b1d3126cb88", size = 1633489, upload-time = "2026-07-22T12:25:09.605Z" }, + { url = "https://files.pythonhosted.org/packages/fa/06/7466ced82818d6132462d7f26b3f83c66ea15d2b193a6c0088d558ed7d95/greenlet-3.5.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2a924f15d17957e252a810acefcb5942f5ca712298e8b6fcaed9a307d357522c", size = 1696584, upload-time = "2026-07-22T11:51:21.304Z" }, + { url = "https://files.pythonhosted.org/packages/f9/4d/55b638489260065de9ffce606c8b5d04507bef705de4b212a0c3d6a1a0df/greenlet-3.5.4-cp315-cp315-win_amd64.whl", hash = "sha256:ed17e5f3420360d5b459de8462efb52060399a5326a613d4cde31cef63ef95da", size = 248297, upload-time = "2026-07-22T11:42:04.055Z" }, + { url = "https://files.pythonhosted.org/packages/bb/08/9dd4ae635da93d41dc268bc34bd62a9d711ed8b8825c5d22ac910c7d6e6d/greenlet-3.5.4-cp315-cp315-win_arm64.whl", hash = "sha256:f908898d6fa484ce4b6f447ce70ea99b52c503fee419e53cf74d60a16bc9e667", size = 247423, upload-time = "2026-07-22T11:44:00.764Z" }, + { url = "https://files.pythonhosted.org/packages/19/66/7c87ed9cdbf1d49c2c6cd1c7b9dd4d16c33b24235ca03972293a1876b30c/greenlet-3.5.4-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:1833637f17d5e7472548a48575c394fe39f1b1890d676d162d86593610f44d8c", size = 306487, upload-time = "2026-07-22T11:41:25.118Z" }, + { url = "https://files.pythonhosted.org/packages/5b/05/0a4201e7c0054866eefc05da234f236dd4c950d0fbf9ca0517141f01b269/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:12cda9122e03341f1cb6b8207a19d7a9d375e52f1b4e9243918375f40fd7b4b9", size = 676479, upload-time = "2026-07-22T12:26:48.129Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c0/4b6b8c5a3aec70f0649cd89662d120fdd6421e2bdc8e3b15c3ab5ec568d8/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d83ae0e32d14957ab7170785a20f582635c8474deab1bfbb552b17e769a6ce25", size = 684321, upload-time = "2026-07-22T12:29:08.925Z" }, + { url = "https://files.pythonhosted.org/packages/88/15/0b167aeea95285b0e654ddce651922f666c089363c2ec528ca8b9a9ba74f/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:123aa379c962ed5fe90a880327e0c3066124ac64ec99e12a238be9fd8eb3db3d", size = 685995, upload-time = "2026-07-22T12:43:50.993Z" }, + { url = "https://files.pythonhosted.org/packages/24/c9/b49c31c9a972eee91e260445770e922244a7efc542697f51a012ec046d0f/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f1467de1bb767f75db0aa34c195e3a496d8d1278c796e70c24ce205d3e99cde", size = 681293, upload-time = "2026-07-22T11:51:20.43Z" }, + { url = "https://files.pythonhosted.org/packages/de/90/c023ec337f32ff505be7db759c80d98f0532bb94d0c6fa13645efe9bee2e/greenlet-3.5.4-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:adf2244d7f69409925a8f22ed22cc5f93cdfe5c9dc87ff3476be2c2aaae61a05", size = 516928, upload-time = "2026-07-22T12:39:58.359Z" }, + { url = "https://files.pythonhosted.org/packages/95/6f/7f2d4653770500eee667866016d42d7a68e3d3462f80df6b8e3fcd48a0eb/greenlet-3.5.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:0fa53040b78b578120eecdc0265e3f1051487cc425d11a2b7c761daadf4feaa8", size = 1642474, upload-time = "2026-07-22T12:25:10.819Z" }, + { url = "https://files.pythonhosted.org/packages/5a/d8/8cba31036a4caae448087ba5d150660ab03b4a0f54d9150f6495a3be7262/greenlet-3.5.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:60e0bc961d367df506660e9ac0177a76bc6d81305300704b0977d1634f76efe2", size = 1701012, upload-time = "2026-07-22T11:51:23.17Z" }, + { url = "https://files.pythonhosted.org/packages/e3/cd/3f77a4cce3bae631b08eb52f53a82a976669600337e21dfdba811cb50267/greenlet-3.5.4-cp315-cp315t-win_amd64.whl", hash = "sha256:f680e549edb3eaf21eea4e7fe101e15ec180c74b7879ab46adc080f22d4015d2", size = 251977, upload-time = "2026-07-22T11:41:38.125Z" }, + { url = "https://files.pythonhosted.org/packages/93/e8/65e8707d00fe2a49bf12f609a9b2b39ba6dd23c2810eacad877c4fc94bfe/greenlet-3.5.4-cp315-cp315t-win_arm64.whl", hash = "sha256:08fc36de8442d5c3e95b044550dbea9bf144d31ec0cc58e36fb241cb6ef6a994", size = 250538, upload-time = "2026-07-22T11:40:17.985Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "hf-xet" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/ab/522a2ab67f27971a9d48ca666d4fca85ef7d5282d142e31fd087e27b1bbe/hf_xet-1.6.0.tar.gz", hash = "sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef", size = 920527, upload-time = "2026-08-03T22:33:13.243Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/62/3c062f593bd92ef4e77a0ef39541e3d82a0a1d3947c8a777a02a13a27828/hf_xet-1.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d", size = 4074584, upload-time = "2026-08-03T22:32:47.364Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1e/c0ad437dd267a8e435bef594acf781bbc3874ff0b6435b4962d03ecf7cc4/hf_xet-1.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675", size = 3867381, upload-time = "2026-08-03T22:32:49.049Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ee/7c0d7b6ab336167531b1c30af2af003f054af4c749becbd7209ae33a77c3/hf_xet-1.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b", size = 4453982, upload-time = "2026-08-03T22:32:50.568Z" }, + { url = "https://files.pythonhosted.org/packages/63/06/ad8eab1c9525246650cbaa821caa3cdbaca734ab1a5b8c91bea09cbd8d69/hf_xet-1.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522", size = 4249445, upload-time = "2026-08-03T22:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/d8/26/1eee8aedb0dafc1ab9717dc9ac602cde33361b232dc06803f1f6ed18b58c/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e", size = 4451099, upload-time = "2026-08-03T22:32:54.114Z" }, + { url = "https://files.pythonhosted.org/packages/67/57/0b88af1f194ab6c9c650547d9cc06bfeaab836ae4dcdb331676bfb8be95a/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9", size = 4664712, upload-time = "2026-08-03T22:32:55.547Z" }, + { url = "https://files.pythonhosted.org/packages/53/a0/26b717a9d1840e8abf48dcec64b5ed8fbe472671d38ad28d30e147132b33/hf_xet-1.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338", size = 4025906, upload-time = "2026-08-03T22:32:57.391Z" }, + { url = "https://files.pythonhosted.org/packages/49/f6/4a9966633c6fef83af997e2cff68ec1963676d412bdfd096df2a93b8e185/hf_xet-1.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765", size = 3849221, upload-time = "2026-08-03T22:32:59.123Z" }, + { url = "https://files.pythonhosted.org/packages/a2/50/7afa2c9c787405864fc47a0d1bbc02c62e9101947ed43c1f43899fc7d91d/hf_xet-1.6.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d", size = 4071729, upload-time = "2026-08-03T22:33:00.721Z" }, + { url = "https://files.pythonhosted.org/packages/4b/69/55b8dcf636142ae660fec1869fcac14c4da2e8412e14d6eee1523be77e9f/hf_xet-1.6.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a", size = 3876287, upload-time = "2026-08-03T22:33:02.251Z" }, + { url = "https://files.pythonhosted.org/packages/67/4e/a28359bf1c1ecf11eba22123168c138698f7cb576ac678f5a2e16cd5da08/hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f", size = 4464663, upload-time = "2026-08-03T22:33:03.802Z" }, + { url = "https://files.pythonhosted.org/packages/9a/69/1f0cbc2fb22ae6082d094f743d1b8945a3f36f6089cb95f42b7ee348cda7/hf_xet-1.6.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7", size = 4262538, upload-time = "2026-08-03T22:33:05.287Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3a/4f4f2301ade26e404462d3336fa11f7958d914cabbabdd6e03c3c5d5658c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb", size = 4460520, upload-time = "2026-08-03T22:33:06.81Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/311725e2a905534dfee2dcb5b08414f249147f1f12252bfc2bd24caa075c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c", size = 4675937, upload-time = "2026-08-03T22:33:08.616Z" }, + { url = "https://files.pythonhosted.org/packages/98/b7/8c59a66d15205024662f1d66968136f13893f96df1ddc5087e2e281fc95f/hf_xet-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b", size = 4033128, upload-time = "2026-08-03T22:33:10.171Z" }, + { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "huggingface" +version = "0.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/a4/168b574a23c1841fab5b24ecac98a88ea626ea3c746c481f79eb360c81f2/huggingface-0.0.1.tar.gz", hash = "sha256:0a2f228fd956801d68b7c6a8bef478dfa60c4b7d7eba572ea7de39ecf87e505a", size = 2320, upload-time = "2020-12-18T18:37:00.599Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/8c/e61fbc39c0a37140e1d4941c4af29e2d53bacf9f4559e3de24d8f4e484f0/huggingface-0.0.1-py3-none-any.whl", hash = "sha256:98a3409537557cd2fd768997ef94cab08529f86c5e106e6d54bbabdd5ee03910", size = 2455, upload-time = "2020-12-18T18:36:59.096Z" }, +] + +[[package]] +name = "huggingface-hub" +version = "1.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/db/3582597f8be0d34bd6881365a26d390854f12893eabdd62dd36de9df5a47/huggingface_hub-1.26.0.tar.gz", hash = "sha256:c8cd4e2df1ba9402f77fce9b509ec1d52debb502551789473f34016acc14e361", size = 936665, upload-time = "2026-07-30T14:12:04.156Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/bb/63a644c75b545f3ff394b822e9bd1c4a9586489c618b77a4d8a44a33a23b/huggingface_hub-1.26.0-py3-none-any.whl", hash = "sha256:e8cca670caa5d8dfa7e45bf45e86b466698198cd8150c021bcdb4a86b9252364", size = 780357, upload-time = "2026-07-30T14:12:01.998Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "imageio" +version = "2.37.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "pillow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/62/aa770a9307508d2a2a2c62d536a49347bffe9e55322db27838d3c93d0b07/imageio-2.37.4.tar.gz", hash = "sha256:e45cbc5e83502047fb138f7f585f7f105a136a57eea5f4b3cfc6ce1b52720bd3", size = 390173, upload-time = "2026-07-20T05:26:11.369Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/2d/ca050652104bab2cf55e569db2a178b1b61cb041fef28307f2db383f6d9f/imageio-2.37.4-py3-none-any.whl", hash = "sha256:1ab2e22c8debf700f24c3ac43e8f95f3b3a8110c83b93411e97b4b0b2cd1c7e6", size = 318000, upload-time = "2026-07-20T05:26:09.874Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "interfaze-complete-benchmarks" +version = "0.1.2" +source = { virtual = "." } +dependencies = [ + { name = "apted" }, + { name = "beautifulsoup4" }, + { name = "datasets" }, + { name = "distance" }, + { name = "editdistance" }, + { name = "evaluate" }, + { name = "func-timeout" }, + { name = "fuzzysearch" }, + { name = "gdown" }, + { name = "google-genai" }, + { name = "huggingface" }, + { name = "ipdb" }, + { name = "jieba" }, + { name = "jiwer" }, + { name = "langchain" }, + { name = "langchain-openai" }, + { name = "levenshtein" }, + { name = "loguru" }, + { name = "lxml" }, + { name = "matplotlib" }, + { name = "mmeval" }, + { name = "nltk" }, + { name = "olmocr" }, + { name = "openai" }, + { name = "opencv-python" }, + { name = "pillow" }, + { name = "playwright" }, + { name = "polygon3" }, + { name = "pycocotools" }, + { name = "pylatexenc" }, + { name = "pymupdf" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "ruff" }, + { name = "scikit-image" }, + { name = "scipy" }, + { name = "tabulate" }, + { name = "tqdm" }, + { name = "zss" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-asyncio" }, +] + +[package.metadata] +requires-dist = [ + { name = "apted", specifier = ">=1.0.3" }, + { name = "beautifulsoup4", specifier = ">=4.11.0" }, + { name = "datasets", specifier = ">=4.7.0" }, + { name = "distance", specifier = ">=0.1.3" }, + { name = "editdistance", specifier = ">=0.8.0" }, + { name = "evaluate", specifier = ">=0.4.0" }, + { name = "func-timeout", specifier = ">=4.3.5" }, + { name = "fuzzysearch", specifier = ">=0.7.3" }, + { name = "gdown", specifier = ">=6.1.0" }, + { name = "google-genai", specifier = ">=2.18.1" }, + { name = "huggingface", specifier = ">=0.0.1" }, + { name = "ipdb", specifier = ">=0.13.0" }, + { name = "ipdb", specifier = ">=0.13.13" }, + { name = "jieba", specifier = ">=0.42.0" }, + { name = "jiwer", specifier = ">=3.0.0" }, + { name = "langchain", specifier = ">=1.2.11" }, + { name = "langchain-openai", specifier = ">=1.1.11" }, + { name = "levenshtein", specifier = ">=0.25.0" }, + { name = "loguru", specifier = ">=0.7.0" }, + { name = "lxml", specifier = ">=5.0.0" }, + { name = "matplotlib", specifier = ">=3.7.0" }, + { name = "mmeval", specifier = ">=0.2.1" }, + { name = "nltk", specifier = ">=3.9.0" }, + { name = "olmocr", specifier = "==0.4.27" }, + { name = "openai", specifier = ">=2.26.0" }, + { name = "opencv-python", specifier = ">=4.10.0" }, + { name = "pillow", specifier = ">=10.0.0" }, + { name = "pillow", specifier = ">=12.1.1" }, + { name = "playwright", specifier = ">=1.44.0" }, + { name = "polygon3", specifier = ">=3.0.0" }, + { name = "polygon3", specifier = ">=3.0.9.1" }, + { name = "pycocotools", specifier = ">=2.0.7" }, + { name = "pylatexenc", specifier = ">=3.0a30" }, + { name = "pymupdf", specifier = ">=1.27.2" }, + { name = "python-dotenv", specifier = ">=1.2.2" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "ruff", specifier = ">=0.15.5" }, + { name = "scikit-image", specifier = ">=0.21.0" }, + { name = "scipy", specifier = ">=1.10.0" }, + { name = "tabulate", specifier = ">=0.9.0" }, + { name = "tqdm", specifier = ">=4.66.0" }, + { name = "zss", specifier = ">=1.2.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.0.0" }, + { name = "pytest-asyncio", specifier = ">=0.24.0" }, +] + +[[package]] +name = "ipdb" +version = "0.13.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "decorator" }, + { name = "ipython" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/1b/7e07e7b752017f7693a0f4d41c13e5ca29ce8cbcfdcc1fd6c4ad8c0a27a0/ipdb-0.13.13.tar.gz", hash = "sha256:e3ac6018ef05126d442af680aad863006ec19d02290561ac88b8b1c0b0cfc726", size = 17042, upload-time = "2023-03-09T15:40:57.487Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/4c/b075da0092003d9a55cf2ecc1cae9384a1ca4f650d51b00fc59875fe76f6/ipdb-0.13.13-py3-none-any.whl", hash = "sha256:45529994741c4ab6d2388bfa5d7b725c2cf7fe9deffabdb8a6113aa5ed449ed4", size = 12130, upload-time = "2023-03-09T15:40:55.021Z" }, +] + +[[package]] +name = "ipython" +version = "9.16.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "psutil", marker = "sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/96/b150fe7e25a5a29ae9ac1374e71488639605d39a1ea4abb74c9ce33af235/ipython-9.16.1.tar.gz", hash = "sha256:5a3d1f9a47ff216d6cf9cf863124f6a2c1a198d1354c546a4d24a370a283b64c", size = 4515302, upload-time = "2026-08-03T08:36:15.571Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/8e/1239df488393d61076653bfb29f759d0f60cab8e030abdf7c17c31539b51/ipython-9.16.1-py3-none-any.whl", hash = "sha256:4acae635506f6d352d94c4899a19d5f85f8bc4d230932342dca556fdab1c69b4", size = 625974, upload-time = "2026-08-03T08:36:13.654Z" }, +] + +[[package]] +name = "ipython-pygments-lexers" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, +] + +[[package]] +name = "jedi" +version = "0.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "parso" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/46/b7/a3635f6a2d7cf5b5dd98064fc1d5fbbafcb25477bcea204a3a92145d158b/jedi-0.20.0.tar.gz", hash = "sha256:c3f4ccbd276696f4b19c54618d4fb18f9fc24b0aef02acf704b23f487daa1011", size = 3119416, upload-time = "2026-05-01T23:38:47.814Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl", hash = "sha256:7bdd9c2634f56713299976f4cbd59cb3fa92165cc5e05ea811fb253480728b67", size = 4884812, upload-time = "2026-05-01T23:38:43.919Z" }, +] + +[[package]] +name = "jieba" +version = "0.42.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c6/cb/18eeb235f833b726522d7ebed54f2278ce28ba9438e3135ab0278d9792a2/jieba-0.42.1.tar.gz", hash = "sha256:055ca12f62674fafed09427f176506079bc135638a14e23e25be909131928db2", size = 19214172, upload-time = "2020-01-20T14:27:23.5Z" } + +[[package]] +name = "jiter" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/1f/10936e16d8860c70698a1aa939a46aa0224813b782bce4e000e637da0b2d/jiter-0.16.0.tar.gz", hash = "sha256:7b24c3492c5f4f84a37946ad9cf504910cf6a782d6a4e0689b6673c5894b4a1c", size = 176431, upload-time = "2026-06-29T13:05:13.657Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/2b/52ace16ed031354f0539749a49e4bf33797d82bea5137910835fa4b09793/jiter-0.16.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:67c3bc1760f8c99d805dcab4e644027142a53b1d5d861f18780ebdbd5d40b72a", size = 306943, upload-time = "2026-06-29T13:03:14.035Z" }, + { url = "https://files.pythonhosted.org/packages/94/2e/34957c2c1b661c252ba9bcc60ae0bddc27e0f7202c6073326a13c5390eec/jiter-0.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5af7780e4a26bd7d0d989592bf9ef12ebf806b74ab709223ecca37c749872ea9", size = 307779, upload-time = "2026-06-29T13:03:15.418Z" }, + { url = "https://files.pythonhosted.org/packages/88/6c/59bd309cab4460c54cf1079f3eb7fe7af6a4c895c5c957a53378693bad2b/jiter-0.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5bf78d0e05e45cfdd66558893938d59afe3d1b1a824a202039b20e607d25a72", size = 335826, upload-time = "2026-06-29T13:03:17.11Z" }, + { url = "https://files.pythonhosted.org/packages/3b/8c/f5ef7b65f0df47afa16596969defb281ebb86e96df346d62be6fd853d620/jiter-0.16.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4444a83f946605990c98f625cdd3d2725bfb818158760c5748c653170a20e0e", size = 362573, upload-time = "2026-06-29T13:03:18.781Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0b/ace4354da061ee38844a0c27dc2c21eecd27aea119e8da324bea987522d0/jiter-0.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3a23f0e4f957e1be65752d2dfac9a5a06b1917af8dc85deb639c3b9d02e31290", size = 457979, upload-time = "2026-06-29T13:03:20.293Z" }, + { url = "https://files.pythonhosted.org/packages/55/40/c0253d3772eb9dcd8e6606ee9b2d53ec8e5b814589c47f140aa585f21eaa/jiter-0.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c22a488f7b9218e245a0025a9ba6b100e2e54700831cf4cf16833a27fba3ad01", size = 372302, upload-time = "2026-06-29T13:03:21.739Z" }, + { url = "https://files.pythonhosted.org/packages/a8/d2/4839422241aa12860ce597b20068727094ba0bc480723c74924ca5bad483/jiter-0.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46add52f4ad47a08bfb1219f3e673da972191489a33016edefdb5ea55bfa8c48", size = 343805, upload-time = "2026-06-29T13:03:23.384Z" }, + { url = "https://files.pythonhosted.org/packages/e2/59/e196888a05befdda7dbe299b722d56f2f6eec65402bc34c0a3306d595feb/jiter-0.16.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:9c8a956fd72c2cf1e730d01ea080341f13aa0a97a4a33b51abebe725b7ae9ca9", size = 351107, upload-time = "2026-06-29T13:03:24.815Z" }, + { url = "https://files.pythonhosted.org/packages/ec/74/4cd9e0fca65232136400354b630fbfcd2de634e22ccbb96567725981b548/jiter-0.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:561926e0573ffe4a32498420a76d64b16c513e1ab413b9d28158a8764ac701e5", size = 388441, upload-time = "2026-06-29T13:03:26.266Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8c/554691e48bc711299c0a293dd8a6179e24b2d66a54dc295421fcf64569c0/jiter-0.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:44d019fa8cdaf89bf29c71b39e3712143fdd0ac76725c6ef954f9957a5ea8730", size = 516354, upload-time = "2026-06-29T13:03:28.02Z" }, + { url = "https://files.pythonhosted.org/packages/a4/cb/01e9d69dc2cc6759d4f91e230b34489c4fdb2518992650633f9e20bece89/jiter-0.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0df91907609837f33341b8e6fe73b95991fdaa57caf1a0fbd343dffe826f386f", size = 547880, upload-time = "2026-06-29T13:03:29.534Z" }, + { url = "https://files.pythonhosted.org/packages/79/70/2953195f1c6ad00f49fa67e13df7e60acb3dd4f387101bc15abccddd905e/jiter-0.16.0-cp312-cp312-win32.whl", hash = "sha256:51d7b836acb0108d7c77df1742332cac2a1fa04a74d6dacec46e7091f0e91274", size = 203473, upload-time = "2026-06-29T13:03:31.025Z" }, + { url = "https://files.pythonhosted.org/packages/2d/05/2909a8b10699a4d560f8c502b6b2c5f3991b682b1922c1eedda242b225bd/jiter-0.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:1878349266f8ee36ecb1375cc5ba2f115f35fd9f0a1a4119e725e379126647f7", size = 196905, upload-time = "2026-06-29T13:03:32.472Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a9/6b82bb1c8d7790d602489b967b982a909e5d092875a6c2ade96444c8dfc5/jiter-0.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:2ed5738ae4af18271a51a528b8811b0cbfa4a1858de9d83359e4169855d6a331", size = 190618, upload-time = "2026-06-29T13:03:34.672Z" }, + { url = "https://files.pythonhosted.org/packages/91/c0/555fc60473d30d66894ba825e63615e3be7524fac23858356afa7a38906c/jiter-0.16.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:41977aa5654023948c2dae2a81cbf9c43343954bef1cd59a154dd15a4d84c195", size = 306203, upload-time = "2026-06-29T13:03:36.243Z" }, + { url = "https://files.pythonhosted.org/packages/d0/2b/c3eaf16f5d7c9bad66ea32f40a95bd169b29a91217fcc7f081375157e99c/jiter-0.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d28bb3c26762358dadf3e5bf0bccd29ae987d65e6988d2e6f49829c76b003c09", size = 306489, upload-time = "2026-06-29T13:03:37.846Z" }, + { url = "https://files.pythonhosted.org/packages/96/3f/02fdfc6705cad96127d883af5c34e4867f554f29ec7705ec1a46156400a9/jiter-0.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0542a7189c26920778658fc8fcf2af8bae05bae9924577f71804acef37996536", size = 335453, upload-time = "2026-06-29T13:03:39.221Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a6/e4bda5920d4b0d7c5dfb7174ce4a6b2e4d3e11c9162c452ef0eab4cdbdbd/jiter-0.16.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8fb8de1e23a0cb2a7f53c335049c7b72b6db41aa6227cdcc0972a1de5cb39450", size = 361625, upload-time = "2026-06-29T13:03:40.597Z" }, + { url = "https://files.pythonhosted.org/packages/b7/97/4e6b59b2c6e55cbb3e183595f81ad65dcfb21c915fee5e19e335df21bc55/jiter-0.16.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b72d0b2990ca754a9102779ac98d8597b7cb31678958562214a007f909eab78e", size = 456958, upload-time = "2026-06-29T13:03:42.074Z" }, + { url = "https://files.pythonhosted.org/packages/15/e0/97e9557686d2f94f4b93786eccb7eed28e9228ad132ea8237f44727314a7/jiter-0.16.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5f91b1c27fc22a57993d5a5cb8a627cb8ed4b10502716fac1ffbfe1d19d84e8", size = 372017, upload-time = "2026-06-29T13:03:43.658Z" }, + { url = "https://files.pythonhosted.org/packages/0f/94/db768b6938e0df35c86beeba3dfbbb025c9ee5c19e1aa271f2396e50864d/jiter-0.16.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c682bea068a90b764577bdb78a60a4c1d1606daf9cd4c893832a37c7cc9d9026", size = 343320, upload-time = "2026-06-29T13:03:45.226Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d6/5a59d938244a30735fe62d9433fd325f9021ea29d89780ea4596ea93bc89/jiter-0.16.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:8d031aabecc4f1b6276adfb42e3aabb77c89d468bf616600e8d3a11328929053", size = 350520, upload-time = "2026-06-29T13:03:46.671Z" }, + { url = "https://files.pythonhosted.org/packages/67/f8/c4a857f49c9af125f6bbcac7e3eee7f7978ed89682833062e2dbf62576b1/jiter-0.16.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eab2cd170150e70153de16896a1774e3a1dca80154c56b54d7a812c479a7165e", size = 387550, upload-time = "2026-06-29T13:03:48.361Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d6/5fbc2f7d6b67b754caa61a993a2e626e815dec47ffc2f9e35f01adfebec7/jiter-0.16.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:6edb63a46e65a82c26800a868e49b2cac30dd5a4218b88d74bc2c848c8ad60bb", size = 515424, upload-time = "2026-06-29T13:03:49.881Z" }, + { url = "https://files.pythonhosted.org/packages/ed/54/284f0164b64a5fed915fea6ba7e9ba9b3d8d37c67d59cf2e3bb99d45cdfe/jiter-0.16.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:659039cc50b5addcc35fcc87ae2c1833b7c0a8e5326ef631a75e4478447bcf84", size = 546981, upload-time = "2026-06-29T13:03:51.363Z" }, + { url = "https://files.pythonhosted.org/packages/13/c5/2a467585a576594384e1d2c43e1224deaafc085f24e243529cf98beef8e1/jiter-0.16.0-cp313-cp313-win32.whl", hash = "sha256:c9c53be232c2e206ef9cdbad81a48bfa74c3d3f08bcf8124630a8a748aad993e", size = 202853, upload-time = "2026-06-29T13:03:53.015Z" }, + { url = "https://files.pythonhosted.org/packages/88/6a/de61d04b9eec69c71719968d2f716532a3bc121170c44a39e14979c6be81/jiter-0.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:baad945ed47f163ad833314f8e3288c396118934f94e7bbb9e243ce4b341a4fd", size = 196160, upload-time = "2026-06-29T13:03:54.447Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/b390ed59bafb3f31d008d1218578f10327714484b334439947f7e5b11e7f/jiter-0.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:3c1fd2dbe1b0af19e987f03fe66c5f5bd105a2229c1aff4ab14890b24f41d21a", size = 189862, upload-time = "2026-06-29T13:03:55.754Z" }, + { url = "https://files.pythonhosted.org/packages/a7/89/bc4f1b57d5da938fd344a466396541e586d161320d70bffd929aaafcd8f4/jiter-0.16.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b2c61484666ad42726029af0c00ef4541f0f3b5cdc550221f56c2343208018ee", size = 308239, upload-time = "2026-06-29T13:03:57.205Z" }, + { url = "https://files.pythonhosted.org/packages/65/7a/c415453e5213001bf3b411ff65dec3d303b0e76a4a2cfea9768cd4960994/jiter-0.16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:63efadc657488f45db1c676d81e704cac2abf3fdb892def1faea61db053127e2", size = 308928, upload-time = "2026-06-29T13:03:58.643Z" }, + { url = "https://files.pythonhosted.org/packages/11/fc/1f4fb7ebf9a724c7741994f4aae18fba1e2f3133df14521a79194952c34a/jiter-0.16.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf0d73f50e7b6935677854f6e8e31d499ca7064dd24734f703e060f5b237d883", size = 336998, upload-time = "2026-06-29T13:04:00.071Z" }, + { url = "https://files.pythonhosted.org/packages/a0/8d/72cadaac05ccfa7cc3a0a2232862e6c72443ca40cf300ba8b57f9f18b69b/jiter-0.16.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf3ea07d9bc8e7d03a9fbc051295462e6dbc295b894fd72457c3136e3e43d898", size = 362112, upload-time = "2026-06-29T13:04:01.52Z" }, + { url = "https://files.pythonhosted.org/packages/58/4a/c4b0d5f651fda90a24ffce9f8d56cde462a2e09d31ae3de3c68cef34c04e/jiter-0.16.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:26798522707abb47d767db536e4148ceac1b14446bf028ee85e579a2e043cfe5", size = 459807, upload-time = "2026-06-29T13:04:03.214Z" }, + { url = "https://files.pythonhosted.org/packages/80/58/ef77879ea9aa56b50824edc5a445e226422c7a8d211f3fd2a56bcb9493cf/jiter-0.16.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bc837c1b9631be10abfe0191537fe8009838204cec7e44827401ace390ddb567", size = 373181, upload-time = "2026-06-29T13:04:04.629Z" }, + { url = "https://files.pythonhosted.org/packages/49/2e/ffbc3f254e4d8a66da3062c624a7df4b7c2b2cf9e1fe43cf394b3e104041/jiter-0.16.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49060fd70737fad59d33ba9dcc0d83247dc9e77187de26053a19c16c9f32bd69", size = 344927, upload-time = "2026-06-29T13:04:06.067Z" }, + { url = "https://files.pythonhosted.org/packages/9a/f6/0be5dc6d64a89f80aa8fec984f94dedb2973e251edcae55841d60786d578/jiter-0.16.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:adbb8edeadd431bc4477879d5d371ece7cb1334486584e0f252656dd7ffada29", size = 352754, upload-time = "2026-06-29T13:04:07.477Z" }, + { url = "https://files.pythonhosted.org/packages/da/6e/7d31243b3b91cd261dd19e9d3557fc3251a80883d3d8049c86174e7ab7af/jiter-0.16.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:31aaee5b80f672c1dc21272bcfb9cbdcfc1ea04ff50f00ed5af500b80c44fa93", size = 390553, upload-time = "2026-06-29T13:04:08.92Z" }, + { url = "https://files.pythonhosted.org/packages/25/33/51ae371fde3c88897520f62b4d5f8b27ad7103e2bb10812ff52195609853/jiter-0.16.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:6722bcef4ffc86c835574b1b2fac6b33b9fb4a889c781e67950e891591f3c55a", size = 516900, upload-time = "2026-06-29T13:04:10.407Z" }, + { url = "https://files.pythonhosted.org/packages/a0/45/6449b3d123ea439ba79507c657288f461d55049e7bcbdc2cf8eb8210f491/jiter-0.16.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:5ab4f50ff971b611d656554ea10b75f80097392c827bc32923c6eeb6386c8b00", size = 548754, upload-time = "2026-06-29T13:04:12.046Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e7/fd2fb11ae3e2649333da3aa170d04d7b3000bbdc3b270f6513382fdf4e04/jiter-0.16.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:710cc51d4ebdcd3c1f70b232c1db1ea1344a075770422bbd4bede5708335acbe", size = 122381, upload-time = "2026-06-29T13:04:13.413Z" }, + { url = "https://files.pythonhosted.org/packages/26/80/f0b147a62c315a164ed2168908286ca302310824c218d3aae52b06c0c9a9/jiter-0.16.0-cp314-cp314-win32.whl", hash = "sha256:57b37fc887a32d44798e4d8ebfa7c9683ff3da1d5bf38f08d1bb3573ccb39106", size = 204578, upload-time = "2026-06-29T13:04:14.813Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e6/4758a14304b4523a6f5adb2419340086aa3593bd4327c2b25b5948a90548/jiter-0.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:cbd18dd5e2df96b580487b5745adf57ef64ad89ba2d9662fc3c19386acce7db8", size = 198154, upload-time = "2026-06-29T13:04:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/26/be/41fa54a2e7ea41d6c99f1dc5b1f0fd4cb474680304b5d268dd518e81da3a/jiter-0.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:a32d2027a9fa67f109ff245a3252ece3ccc32cc56703e1deab6cc846a59e0585", size = 191458, upload-time = "2026-06-29T13:04:17.707Z" }, + { url = "https://files.pythonhosted.org/packages/81/6b/59127338b86d9fe4d99418f5a15118bea778103ee0fe9d9dd7e0af174e95/jiter-0.16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2577196f4474ef3fc4779a088a23b0897bbf86f9ea3679c372d45b8383b43207", size = 316739, upload-time = "2026-06-29T13:04:19.663Z" }, + { url = "https://files.pythonhosted.org/packages/2d/95/49461034d5388196d3dabf98748935f017b7785d8f3f5349f834bcc4ed0d/jiter-0.16.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616e89e008a93c01104161c75b4988e58716b01d62307ebfe161e52a56d2a818", size = 340911, upload-time = "2026-06-29T13:04:21.257Z" }, + { url = "https://files.pythonhosted.org/packages/cd/97/a4369f2fb82cb3dda13b98622f31249b2e014b223fe64ee534413ad72294/jiter-0.16.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e2e9efbe042210df657bade597f66d6d75723e3d8f45a12ea6d8167ff8bbce3", size = 361747, upload-time = "2026-06-29T13:04:22.677Z" }, + { url = "https://files.pythonhosted.org/packages/28/51/49b6ed456261646e1906016a6760367a28aacd3c24805e4e5fe64116c1db/jiter-0.16.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f4d9e473a5ce7d27fef8b848df4dc16e283893d3f53b4a585e72c9595f3c284", size = 460225, upload-time = "2026-06-29T13:04:24.441Z" }, + { url = "https://files.pythonhosted.org/packages/33/b5/5689aff4f66c5b60be63106e591dbfcba2190df97d2c9c7cf052361ddb98/jiter-0.16.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d30a4a1c87713060c8d1cc59a7b6c8fb6b8ef0a6900368014c76c87922a2929", size = 373169, upload-time = "2026-06-29T13:04:25.884Z" }, + { url = "https://files.pythonhosted.org/packages/a2/96/3ae1b85ee0d6d6cab254fb7f8da018272b932bbf2d69b07e98aa2a96c746/jiter-0.16.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bae96332410f866e5900d809298b1ed82735932986c672495f9701daacd80620", size = 350332, upload-time = "2026-06-29T13:04:27.302Z" }, + { url = "https://files.pythonhosted.org/packages/15/32/c99d7bafd78986556c95bf60ce84c6cc98786eac56066c12d7f828bb6747/jiter-0.16.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:da3d7ec75dc83bb18bca888b5edfae0656a26849056c59e05a7728badd17e7af", size = 353377, upload-time = "2026-06-29T13:04:28.731Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4b/f99a8e571287c3dec766bcc18528bbe8e8fb5365522ab5e6d64c93e87066/jiter-0.16.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ee6162b77d49a9939229df666dfa8af3e656b6701b54c4c84966d740e189264e", size = 387746, upload-time = "2026-06-29T13:04:30.319Z" }, + { url = "https://files.pythonhosted.org/packages/75/69/c78a5b3f71040e34eb5917df26fb7ae9a2174cad1ccbf277512507c53a6e/jiter-0.16.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:63ffdbdae7d4499f4cda14eadc12ddcabef0fc0c081191bdc2247489cb698077", size = 517292, upload-time = "2026-06-29T13:04:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f7/095b38eda4c70d03651c403f29a5590f16d12ddc5d544aac9f9cddf72277/jiter-0.16.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a111256a7193bea0759267b10385e5870949c239ed7b6ddbaaf57573edb38734", size = 549259, upload-time = "2026-06-29T13:04:33.721Z" }, + { url = "https://files.pythonhosted.org/packages/2e/c5/6a0207d90e5f656d95af98ebd0934f382d37674416f215aeda2ff8063e51/jiter-0.16.0-cp314-cp314t-win32.whl", hash = "sha256:de5ba8763e56b793561f43bed197c9ea55776daa5e9a6b91eed68a909bc9cdbf", size = 206523, upload-time = "2026-06-29T13:04:35.068Z" }, + { url = "https://files.pythonhosted.org/packages/a5/31/c757d5f30a8980fd945ce7b98be10be9e4ff59c7c42f5fd86804c2e87db8/jiter-0.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b8a3f9a6008048fe9def7bf465180564a6e458047d2ce499149cfbe73c3ae9db", size = 200366, upload-time = "2026-06-29T13:04:36.61Z" }, + { url = "https://files.pythonhosted.org/packages/7c/a2/d88de6d313d734a544a7901353ad5db67cb38dcfcd91713b7979dafc345d/jiter-0.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0fa25b09b13075c46f5bc174f2690525a925a4fc2f7c82969a2bbabff22386ce", size = 190516, upload-time = "2026-06-29T13:04:38.004Z" }, + { url = "https://files.pythonhosted.org/packages/98/ab/664fd8c4be028b2bedd3d2ff08769c4ede23d0dbc87a77c62384a0515b5d/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:f17d61a28b4b3e0e3e2ba98490c70501403b4d196f78732439160e7fd3678127", size = 303106, upload-time = "2026-06-29T13:05:07.118Z" }, + { url = "https://files.pythonhosted.org/packages/1a/07/421f1d5b65493a76e16027b848aba6a7d28073ae75944fa4289cc914d39f/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:96e38eea538c8ddf853a35727c7be0741c76c13f04148ac5c116222f50ece3b3", size = 304658, upload-time = "2026-06-29T13:05:08.708Z" }, + { url = "https://files.pythonhosted.org/packages/0a/db/bba1155f01a01c3c37a89425d571da751bbedf5c54247b831a04cb971798/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d284fb8d94d5855d60c44fefcab4bf966f1da6fada73992b01f6f0c9bc0c6702", size = 339719, upload-time = "2026-06-29T13:05:10.41Z" }, + { url = "https://files.pythonhosted.org/packages/78/f7/18a1afcd64f35314b68c1f23afcd9994d0bc13e65cc77517afff4e83986d/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d613743df53199b1aa256a7d328340da6d7078aac7705a7db9d7a791e9cfd2", size = 343885, upload-time = "2026-06-29T13:05:12.087Z" }, +] + +[[package]] +name = "jiwer" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "rapidfuzz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/12/1e/963dfc249d5bbe88079b57a22556e981ddd9208e4b6116e48bdbfc01f26b/jiwer-4.0.0.tar.gz", hash = "sha256:ae9c051469102a61ef0927100baeeb4546f78d180c9b0948281d08eaf44c191e", size = 28074, upload-time = "2025-06-19T16:05:23.004Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/c9/172c525330c739a068c01050759a6f855ce16212db10a0359e690a03ac48/jiwer-4.0.0-py3-none-any.whl", hash = "sha256:7efaf0bd336b095d99ddef9dd67e1ee829d75d58aa2a81d9639870b01d6d95ea", size = 23034, upload-time = "2025-06-19T16:05:21.821Z" }, +] + +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + +[[package]] +name = "joblib" +version = "1.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, +] + +[[package]] +name = "jsonpatch" +version = "1.33" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonpointer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/78/18813351fe5d63acad16aec57f94ec2b70a09e53ca98145589e185423873/jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c", size = 21699, upload-time = "2023-06-26T12:07:29.144Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade", size = 12898, upload-time = "2023-06-16T21:01:28.466Z" }, +] + +[[package]] +name = "jsonpointer" +version = "3.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/c7/af399a2e7a67fd18d63c40c5e62d3af4e67b836a2107468b6a5ea24c4304/jsonpointer-3.1.1.tar.gz", hash = "sha256:0b801c7db33a904024f6004d526dcc53bbb8a4a0f4e32bfd10beadf60adf1900", size = 9068, upload-time = "2026-03-23T22:32:32.458Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/6a/a83720e953b1682d2d109d3c2dbb0bc9bf28cc1cbc205be4ef4be5da709d/jsonpointer-3.1.1-py3-none-any.whl", hash = "sha256:8ff8b95779d071ba472cf5bc913028df06031797532f08a7d5b602d8b2a488ca", size = 7659, upload-time = "2026-03-23T22:32:31.568Z" }, +] + +[[package]] +name = "kiwisolver" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/67/9c61eccb13f0bdca9307614e782fec49ffdde0f7a2314935d489fa93cd9c/kiwisolver-1.5.0.tar.gz", hash = "sha256:d4193f3d9dc3f6f79aaed0e5637f45d98850ebf01f7ca20e69457f3e8946b66a", size = 103482, upload-time = "2026-03-09T13:15:53.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/b2/818b74ebea34dabe6d0c51cb1c572e046730e64844da6ed646d5298c40ce/kiwisolver-1.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:4e9750bc21b886308024f8a54ccb9a2cc38ac9fa813bf4348434e3d54f337ff9", size = 123158, upload-time = "2026-03-09T13:13:23.127Z" }, + { url = "https://files.pythonhosted.org/packages/bf/d9/405320f8077e8e1c5c4bd6adc45e1e6edf6d727b6da7f2e2533cf58bff71/kiwisolver-1.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:72ec46b7eba5b395e0a7b63025490d3214c11013f4aacb4f5e8d6c3041829588", size = 66388, upload-time = "2026-03-09T13:13:24.765Z" }, + { url = "https://files.pythonhosted.org/packages/99/9f/795fedf35634f746151ca8839d05681ceb6287fbed6cc1c9bf235f7887c2/kiwisolver-1.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ed3a984b31da7481b103f68776f7128a89ef26ed40f4dc41a2223cda7fb24819", size = 64068, upload-time = "2026-03-09T13:13:25.878Z" }, + { url = "https://files.pythonhosted.org/packages/c4/13/680c54afe3e65767bed7ec1a15571e1a2f1257128733851ade24abcefbcc/kiwisolver-1.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb5136fb5352d3f422df33f0c879a1b0c204004324150cc3b5e3c4f310c9049f", size = 1477934, upload-time = "2026-03-09T13:13:27.166Z" }, + { url = "https://files.pythonhosted.org/packages/c8/2f/cebfcdb60fd6a9b0f6b47a9337198bcbad6fbe15e68189b7011fd914911f/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b2af221f268f5af85e776a73d62b0845fc8baf8ef0abfae79d29c77d0e776aaf", size = 1278537, upload-time = "2026-03-09T13:13:28.707Z" }, + { url = "https://files.pythonhosted.org/packages/f2/0d/9b782923aada3fafb1d6b84e13121954515c669b18af0c26e7d21f579855/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b0f172dc8ffaccb8522d7c5d899de00133f2f1ca7b0a49b7da98e901de87bf2d", size = 1296685, upload-time = "2026-03-09T13:13:30.528Z" }, + { url = "https://files.pythonhosted.org/packages/27/70/83241b6634b04fe44e892688d5208332bde130f38e610c0418f9ede47ded/kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6ab8ba9152203feec73758dad83af9a0bbe05001eb4639e547207c40cfb52083", size = 1346024, upload-time = "2026-03-09T13:13:32.818Z" }, + { url = "https://files.pythonhosted.org/packages/e4/db/30ed226fb271ae1a6431fc0fe0edffb2efe23cadb01e798caeb9f2ceae8f/kiwisolver-1.5.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:cdee07c4d7f6d72008d3f73b9bf027f4e11550224c7c50d8df1ae4a37c1402a6", size = 987241, upload-time = "2026-03-09T13:13:34.435Z" }, + { url = "https://files.pythonhosted.org/packages/ec/bd/c314595208e4c9587652d50959ead9e461995389664e490f4dce7ff0f782/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7c60d3c9b06fb23bd9c6139281ccbdc384297579ae037f08ae90c69f6845c0b1", size = 2227742, upload-time = "2026-03-09T13:13:36.4Z" }, + { url = "https://files.pythonhosted.org/packages/c1/43/0499cec932d935229b5543d073c2b87c9c22846aab48881e9d8d6e742a2d/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e315e5ec90d88e140f57696ff85b484ff68bb311e36f2c414aa4286293e6dee0", size = 2323966, upload-time = "2026-03-09T13:13:38.204Z" }, + { url = "https://files.pythonhosted.org/packages/3d/6f/79b0d760907965acfd9d61826a3d41f8f093c538f55cd2633d3f0db269f6/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:1465387ac63576c3e125e5337a6892b9e99e0627d52317f3ca79e6930d889d15", size = 1977417, upload-time = "2026-03-09T13:13:39.966Z" }, + { url = "https://files.pythonhosted.org/packages/ab/31/01d0537c41cb75a551a438c3c7a80d0c60d60b81f694dac83dd436aec0d0/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:530a3fd64c87cffa844d4b6b9768774763d9caa299e9b75d8eca6a4423b31314", size = 2491238, upload-time = "2026-03-09T13:13:41.698Z" }, + { url = "https://files.pythonhosted.org/packages/e4/34/8aefdd0be9cfd00a44509251ba864f5caf2991e36772e61c408007e7f417/kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1d9daea4ea6b9be74fe2f01f7fbade8d6ffab263e781274cffca0dba9be9eec9", size = 2294947, upload-time = "2026-03-09T13:13:43.343Z" }, + { url = "https://files.pythonhosted.org/packages/ad/cf/0348374369ca588f8fe9c338fae49fa4e16eeb10ffb3d012f23a54578a9e/kiwisolver-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:f18c2d9782259a6dc132fdc7a63c168cbc74b35284b6d75c673958982a378384", size = 73569, upload-time = "2026-03-09T13:13:45.792Z" }, + { url = "https://files.pythonhosted.org/packages/28/26/192b26196e2316e2bd29deef67e37cdf9870d9af8e085e521afff0fed526/kiwisolver-1.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:f7c7553b13f69c1b29a5bde08ddc6d9d0c8bfb84f9ed01c30db25944aeb852a7", size = 64997, upload-time = "2026-03-09T13:13:46.878Z" }, + { url = "https://files.pythonhosted.org/packages/9d/69/024d6711d5ba575aa65d5538042e99964104e97fa153a9f10bc369182bc2/kiwisolver-1.5.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:fd40bb9cd0891c4c3cb1ddf83f8bbfa15731a248fdc8162669405451e2724b09", size = 123166, upload-time = "2026-03-09T13:13:48.032Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/adbb40df306f587054a348831220812b9b1d787aff714cfbc8556e38fccd/kiwisolver-1.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c0e1403fd7c26d77c1f03e096dc58a5c726503fa0db0456678b8668f76f521e3", size = 66395, upload-time = "2026-03-09T13:13:49.365Z" }, + { url = "https://files.pythonhosted.org/packages/a8/3a/d0a972b34e1c63e2409413104216cd1caa02c5a37cb668d1687d466c1c45/kiwisolver-1.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dda366d548e89a90d88a86c692377d18d8bd64b39c1fb2b92cb31370e2896bbd", size = 64065, upload-time = "2026-03-09T13:13:50.562Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0a/7b98e1e119878a27ba8618ca1e18b14f992ff1eda40f47bccccf4de44121/kiwisolver-1.5.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:332b4f0145c30b5f5ad9374881133e5aa64320428a57c2c2b61e9d891a51c2f3", size = 1477903, upload-time = "2026-03-09T13:13:52.084Z" }, + { url = "https://files.pythonhosted.org/packages/18/d8/55638d89ffd27799d5cc3d8aa28e12f4ce7a64d67b285114dbedc8ea4136/kiwisolver-1.5.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c50b89ffd3e1a911c69a1dd3de7173c0cd10b130f56222e57898683841e4f96", size = 1278751, upload-time = "2026-03-09T13:13:54.673Z" }, + { url = "https://files.pythonhosted.org/packages/b8/97/b4c8d0d18421ecceba20ad8701358453b88e32414e6f6950b5a4bad54e65/kiwisolver-1.5.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4db576bb8c3ef9365f8b40fe0f671644de6736ae2c27a2c62d7d8a1b4329f099", size = 1296793, upload-time = "2026-03-09T13:13:56.287Z" }, + { url = "https://files.pythonhosted.org/packages/c4/10/f862f94b6389d8957448ec9df59450b81bec4abb318805375c401a1e6892/kiwisolver-1.5.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0b85aad90cea8ac6797a53b5d5f2e967334fa4d1149f031c4537569972596cb8", size = 1346041, upload-time = "2026-03-09T13:13:58.269Z" }, + { url = "https://files.pythonhosted.org/packages/a3/6a/f1650af35821eaf09de398ec0bc2aefc8f211f0cda50204c9f1673741ba9/kiwisolver-1.5.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:d36ca54cb4c6c4686f7cbb7b817f66f5911c12ddb519450bbe86707155028f87", size = 987292, upload-time = "2026-03-09T13:13:59.871Z" }, + { url = "https://files.pythonhosted.org/packages/de/19/d7fb82984b9238115fe629c915007be608ebd23dc8629703d917dbfaffd4/kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:38f4a703656f493b0ad185211ccfca7f0386120f022066b018eb5296d8613e23", size = 2227865, upload-time = "2026-03-09T13:14:01.401Z" }, + { url = "https://files.pythonhosted.org/packages/7f/b9/46b7f386589fd222dac9e9de9c956ce5bcefe2ee73b4e79891381dda8654/kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3ac2360e93cb41be81121755c6462cff3beaa9967188c866e5fce5cf13170859", size = 2324369, upload-time = "2026-03-09T13:14:02.972Z" }, + { url = "https://files.pythonhosted.org/packages/92/8b/95e237cf3d9c642960153c769ddcbe278f182c8affb20cecc1cc983e7cc5/kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c95cab08d1965db3d84a121f1c7ce7479bdd4072c9b3dafd8fecce48a2e6b902", size = 1977989, upload-time = "2026-03-09T13:14:04.503Z" }, + { url = "https://files.pythonhosted.org/packages/1b/95/980c9df53501892784997820136c01f62bc1865e31b82b9560f980c0e649/kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fc20894c3d21194d8041a28b65622d5b86db786da6e3cfe73f0c762951a61167", size = 2491645, upload-time = "2026-03-09T13:14:06.106Z" }, + { url = "https://files.pythonhosted.org/packages/cb/32/900647fd0840abebe1561792c6b31e6a7c0e278fc3973d30572a965ca14c/kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7a32f72973f0f950c1920475d5c5ea3d971b81b6f0ec53b8d0a956cc965f22e0", size = 2295237, upload-time = "2026-03-09T13:14:08.891Z" }, + { url = "https://files.pythonhosted.org/packages/be/8a/be60e3bbcf513cc5a50f4a3e88e1dcecebb79c1ad607a7222877becaa101/kiwisolver-1.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:0bf3acf1419fa93064a4c2189ac0b58e3be7872bf6ee6177b0d4c63dc4cea276", size = 73573, upload-time = "2026-03-09T13:14:12.327Z" }, + { url = "https://files.pythonhosted.org/packages/4d/d2/64be2e429eb4fca7f7e1c52a91b12663aeaf25de3895e5cca0f47ef2a8d0/kiwisolver-1.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:fa8eb9ecdb7efb0b226acec134e0d709e87a909fa4971a54c0c4f6e88635484c", size = 64998, upload-time = "2026-03-09T13:14:13.469Z" }, + { url = "https://files.pythonhosted.org/packages/b0/69/ce68dd0c85755ae2de490bf015b62f2cea5f6b14ff00a463f9d0774449ff/kiwisolver-1.5.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db485b3847d182b908b483b2ed133c66d88d49cacf98fd278fadafe11b4478d1", size = 125700, upload-time = "2026-03-09T13:14:14.636Z" }, + { url = "https://files.pythonhosted.org/packages/74/aa/937aac021cf9d4349990d47eb319309a51355ed1dbdc9c077cdc9224cb11/kiwisolver-1.5.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:be12f931839a3bdfe28b584db0e640a65a8bcbc24560ae3fdb025a449b3d754e", size = 67537, upload-time = "2026-03-09T13:14:15.808Z" }, + { url = "https://files.pythonhosted.org/packages/ee/20/3a87fbece2c40ad0f6f0aefa93542559159c5f99831d596050e8afae7a9f/kiwisolver-1.5.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:16b85d37c2cbb3253226d26e64663f755d88a03439a9c47df6246b35defbdfb7", size = 65514, upload-time = "2026-03-09T13:14:18.035Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7f/f943879cda9007c45e1f7dba216d705c3a18d6b35830e488b6c6a4e7cdf0/kiwisolver-1.5.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4432b835675f0ea7414aab3d37d119f7226d24869b7a829caeab49ebda407b0c", size = 1584848, upload-time = "2026-03-09T13:14:19.745Z" }, + { url = "https://files.pythonhosted.org/packages/37/f8/4d4f85cc1870c127c88d950913370dd76138482161cd07eabbc450deff01/kiwisolver-1.5.0-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b0feb50971481a2cc44d94e88bdb02cdd497618252ae226b8eb1201b957e368", size = 1391542, upload-time = "2026-03-09T13:14:21.54Z" }, + { url = "https://files.pythonhosted.org/packages/04/0b/65dd2916c84d252b244bd405303220f729e7c17c9d7d33dca6feeff9ffc4/kiwisolver-1.5.0-cp313-cp313t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56fa888f10d0f367155e76ce849fa1166fc9730d13bd2d65a2aa13b6f5424489", size = 1404447, upload-time = "2026-03-09T13:14:23.205Z" }, + { url = "https://files.pythonhosted.org/packages/39/5c/2606a373247babce9b1d056c03a04b65f3cf5290a8eac5d7bdead0a17e21/kiwisolver-1.5.0-cp313-cp313t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:940dda65d5e764406b9fb92761cbf462e4e63f712ab60ed98f70552e496f3bf1", size = 1455918, upload-time = "2026-03-09T13:14:24.74Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d1/c6078b5756670658e9192a2ef11e939c92918833d2745f85cd14a6004bdf/kiwisolver-1.5.0-cp313-cp313t-manylinux_2_39_riscv64.whl", hash = "sha256:89fc958c702ee9a745e4700378f5d23fddbc46ff89e8fdbf5395c24d5c1452a3", size = 1072856, upload-time = "2026-03-09T13:14:26.597Z" }, + { url = "https://files.pythonhosted.org/packages/cb/c8/7def6ddf16eb2b3741d8b172bdaa9af882b03c78e9b0772975408801fa63/kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9027d773c4ff81487181a925945743413f6069634d0b122d0b37684ccf4f1e18", size = 2333580, upload-time = "2026-03-09T13:14:28.237Z" }, + { url = "https://files.pythonhosted.org/packages/9e/87/2ac1fce0eb1e616fcd3c35caa23e665e9b1948bb984f4764790924594128/kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:5b233ea3e165e43e35dba1d2b8ecc21cf070b45b65ae17dd2747d2713d942021", size = 2423018, upload-time = "2026-03-09T13:14:30.018Z" }, + { url = "https://files.pythonhosted.org/packages/67/13/c6700ccc6cc218716bfcda4935e4b2997039869b4ad8a94f364c5a3b8e63/kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ce9bf03dad3b46408c08649c6fbd6ca28a9fce0eb32fdfffa6775a13103b5310", size = 2062804, upload-time = "2026-03-09T13:14:32.888Z" }, + { url = "https://files.pythonhosted.org/packages/1b/bd/877056304626943ff0f1f44c08f584300c199b887cb3176cd7e34f1515f1/kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:fc4d3f1fb9ca0ae9f97b095963bc6326f1dbfd3779d6679a1e016b9baaa153d3", size = 2597482, upload-time = "2026-03-09T13:14:34.971Z" }, + { url = "https://files.pythonhosted.org/packages/75/19/c60626c47bf0f8ac5dcf72c6c98e266d714f2fbbfd50cf6dab5ede3aaa50/kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f443b4825c50a51ee68585522ab4a1d1257fac65896f282b4c6763337ac9f5d2", size = 2394328, upload-time = "2026-03-09T13:14:36.816Z" }, + { url = "https://files.pythonhosted.org/packages/47/84/6a6d5e5bb8273756c27b7d810d47f7ef2f1f9b9fd23c9ee9a3f8c75c9cef/kiwisolver-1.5.0-cp313-cp313t-win_arm64.whl", hash = "sha256:893ff3a711d1b515ba9da14ee090519bad4610ed1962fbe298a434e8c5f8db53", size = 68410, upload-time = "2026-03-09T13:14:38.695Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/060f45052f2a01ad5762c8fdecd6d7a752b43400dc29ff75cd47225a40fd/kiwisolver-1.5.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8df31fe574b8b3993cc61764f40941111b25c2d9fea13d3ce24a49907cd2d615", size = 123231, upload-time = "2026-03-09T13:14:41.323Z" }, + { url = "https://files.pythonhosted.org/packages/c2/a7/78da680eadd06ff35edef6ef68a1ad273bad3e2a0936c9a885103230aece/kiwisolver-1.5.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1d49a49ac4cbfb7c1375301cd1ec90169dfeae55ff84710d782260ce77a75a02", size = 66489, upload-time = "2026-03-09T13:14:42.534Z" }, + { url = "https://files.pythonhosted.org/packages/49/b2/97980f3ad4fae37dd7fe31626e2bf75fbf8bdf5d303950ec1fab39a12da8/kiwisolver-1.5.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0cbe94b69b819209a62cb27bdfa5dc2a8977d8de2f89dfd97ba4f53ed3af754e", size = 64063, upload-time = "2026-03-09T13:14:44.759Z" }, + { url = "https://files.pythonhosted.org/packages/e7/f9/b06c934a6aa8bc91f566bd2a214fd04c30506c2d9e2b6b171953216a65b6/kiwisolver-1.5.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:80aa065ffd378ff784822a6d7c3212f2d5f5e9c3589614b5c228b311fd3063ac", size = 1475913, upload-time = "2026-03-09T13:14:46.247Z" }, + { url = "https://files.pythonhosted.org/packages/6b/f0/f768ae564a710135630672981231320bc403cf9152b5596ec5289de0f106/kiwisolver-1.5.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e7f886f47ab881692f278ae901039a234e4025a68e6dfab514263a0b1c4ae05", size = 1282782, upload-time = "2026-03-09T13:14:48.458Z" }, + { url = "https://files.pythonhosted.org/packages/e2/9f/1de7aad00697325f05238a5f2eafbd487fb637cc27a558b5367a5f37fb7f/kiwisolver-1.5.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5060731cc3ed12ca3a8b57acd4aeca5bbc2f49216dd0bec1650a1acd89486bcd", size = 1300815, upload-time = "2026-03-09T13:14:50.721Z" }, + { url = "https://files.pythonhosted.org/packages/5a/c2/297f25141d2e468e0ce7f7a7b92e0cf8918143a0cbd3422c1ad627e85a06/kiwisolver-1.5.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7a4aa69609f40fce3cbc3f87b2061f042eee32f94b8f11db707b66a26461591a", size = 1347925, upload-time = "2026-03-09T13:14:52.304Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d3/f4c73a02eb41520c47610207b21afa8cdd18fdbf64ffd94674ae21c4812d/kiwisolver-1.5.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:d168fda2dbff7b9b5f38e693182d792a938c31db4dac3a80a4888de603c99554", size = 991322, upload-time = "2026-03-09T13:14:54.637Z" }, + { url = "https://files.pythonhosted.org/packages/7b/46/d3f2efef7732fcda98d22bf4ad5d3d71d545167a852ca710a494f4c15343/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:413b820229730d358efd838ecbab79902fe97094565fdc80ddb6b0a18c18a581", size = 2232857, upload-time = "2026-03-09T13:14:56.471Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ec/2d9756bf2b6d26ae4349b8d3662fb3993f16d80c1f971c179ce862b9dbae/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:5124d1ea754509b09e53738ec185584cc609aae4a3b510aaf4ed6aa047ef9303", size = 2329376, upload-time = "2026-03-09T13:14:58.072Z" }, + { url = "https://files.pythonhosted.org/packages/8f/9f/876a0a0f2260f1bde92e002b3019a5fabc35e0939c7d945e0fa66185eb20/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e4415a8db000bf49a6dd1c478bf70062eaacff0f462b92b0ba68791a905861f9", size = 1982549, upload-time = "2026-03-09T13:14:59.668Z" }, + { url = "https://files.pythonhosted.org/packages/6c/4f/ba3624dfac23a64d54ac4179832860cb537c1b0af06024936e82ca4154a0/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d618fd27420381a4f6044faa71f46d8bfd911bd077c555f7138ed88729bfbe79", size = 2494680, upload-time = "2026-03-09T13:15:01.364Z" }, + { url = "https://files.pythonhosted.org/packages/39/b7/97716b190ab98911b20d10bf92eca469121ec483b8ce0edd314f51bc85af/kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5092eb5b1172947f57d6ea7d89b2f29650414e4293c47707eb499ec07a0ac796", size = 2297905, upload-time = "2026-03-09T13:15:03.925Z" }, + { url = "https://files.pythonhosted.org/packages/a3/36/4e551e8aa55c9188bca9abb5096805edbf7431072b76e2298e34fd3a3008/kiwisolver-1.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:d76e2d8c75051d58177e762164d2e9ab92886534e3a12e795f103524f221dd8e", size = 75086, upload-time = "2026-03-09T13:15:07.775Z" }, + { url = "https://files.pythonhosted.org/packages/70/15/9b90f7df0e31a003c71649cf66ef61c3c1b862f48c81007fa2383c8bd8d7/kiwisolver-1.5.0-cp314-cp314-win_arm64.whl", hash = "sha256:fa6248cd194edff41d7ea9425ced8ca3a6f838bfb295f6f1d6e6bb694a8518df", size = 66577, upload-time = "2026-03-09T13:15:09.139Z" }, + { url = "https://files.pythonhosted.org/packages/17/01/7dc8c5443ff42b38e72731643ed7cf1ed9bf01691ae5cdca98501999ed83/kiwisolver-1.5.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d1ffeb80b5676463d7a7d56acbe8e37a20ce725570e09549fe738e02ca6b7e1e", size = 125794, upload-time = "2026-03-09T13:15:10.525Z" }, + { url = "https://files.pythonhosted.org/packages/46/8a/b4ebe46ebaac6a303417fab10c2e165c557ddaff558f9699d302b256bc53/kiwisolver-1.5.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:bc4d8e252f532ab46a1de9349e2d27b91fce46736a9eedaa37beaca66f574ed4", size = 67646, upload-time = "2026-03-09T13:15:12.016Z" }, + { url = "https://files.pythonhosted.org/packages/60/35/10a844afc5f19d6f567359bf4789e26661755a2f36200d5d1ed8ad0126e5/kiwisolver-1.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6783e069732715ad0c3ce96dbf21dbc2235ab0593f2baf6338101f70371f4028", size = 65511, upload-time = "2026-03-09T13:15:13.311Z" }, + { url = "https://files.pythonhosted.org/packages/f8/8a/685b297052dd041dcebce8e8787b58923b6e78acc6115a0dc9189011c44b/kiwisolver-1.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7c4c09a490dc4d4a7f8cbee56c606a320f9dc28cf92a7157a39d1ce7676a657", size = 1584858, upload-time = "2026-03-09T13:15:15.103Z" }, + { url = "https://files.pythonhosted.org/packages/9e/80/04865e3d4638ac5bddec28908916df4a3075b8c6cc101786a96803188b96/kiwisolver-1.5.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a075bd7bd19c70cf67c8badfa36cf7c5d8de3c9ddb8420c51e10d9c50e94920", size = 1392539, upload-time = "2026-03-09T13:15:16.661Z" }, + { url = "https://files.pythonhosted.org/packages/ba/01/77a19cacc0893fa13fafa46d1bba06fb4dc2360b3292baf4b56d8e067b24/kiwisolver-1.5.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bdd3e53429ff02aa319ba59dfe4ceeec345bf46cf180ec2cf6fd5b942e7975e9", size = 1405310, upload-time = "2026-03-09T13:15:18.229Z" }, + { url = "https://files.pythonhosted.org/packages/53/39/bcaf5d0cca50e604cfa9b4e3ae1d64b50ca1ae5b754122396084599ef903/kiwisolver-1.5.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cdcb35dc9d807259c981a85531048ede628eabcffb3239adf3d17463518992d", size = 1456244, upload-time = "2026-03-09T13:15:20.444Z" }, + { url = "https://files.pythonhosted.org/packages/d0/7a/72c187abc6975f6978c3e39b7cf67aeb8b3c0a8f9790aa7fd412855e9e1f/kiwisolver-1.5.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:70d593af6a6ca332d1df73d519fddb5148edb15cd90d5f0155e3746a6d4fcc65", size = 1073154, upload-time = "2026-03-09T13:15:22.039Z" }, + { url = "https://files.pythonhosted.org/packages/c7/ca/cf5b25783ebbd59143b4371ed0c8428a278abe68d6d0104b01865b1bbd0f/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:377815a8616074cabbf3f53354e1d040c35815a134e01d7614b7692e4bf8acfa", size = 2334377, upload-time = "2026-03-09T13:15:23.741Z" }, + { url = "https://files.pythonhosted.org/packages/4a/e5/b1f492adc516796e88751282276745340e2a72dcd0d36cf7173e0daf3210/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0255a027391d52944eae1dbb5d4cc5903f57092f3674e8e544cdd2622826b3f0", size = 2425288, upload-time = "2026-03-09T13:15:25.789Z" }, + { url = "https://files.pythonhosted.org/packages/e6/e5/9b21fbe91a61b8f409d74a26498706e97a48008bfcd1864373d32a6ba31c/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:012b1eb16e28718fa782b5e61dc6f2da1f0792ca73bd05d54de6cb9561665fc9", size = 2063158, upload-time = "2026-03-09T13:15:27.63Z" }, + { url = "https://files.pythonhosted.org/packages/b1/02/83f47986138310f95ea95531f851b2a62227c11cbc3e690ae1374fe49f0f/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:0e3aafb33aed7479377e5e9a82e9d4bf87063741fc99fc7ae48b0f16e32bdd6f", size = 2597260, upload-time = "2026-03-09T13:15:29.421Z" }, + { url = "https://files.pythonhosted.org/packages/07/18/43a5f24608d8c313dd189cf838c8e68d75b115567c6279de7796197cfb6a/kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e7a116ae737f0000343218c4edf5bd45893bfeaff0993c0b215d7124c9f77646", size = 2394403, upload-time = "2026-03-09T13:15:31.517Z" }, + { url = "https://files.pythonhosted.org/packages/3b/b5/98222136d839b8afabcaa943b09bd05888c2d36355b7e448550211d1fca4/kiwisolver-1.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1dd9b0b119a350976a6d781e7278ec7aca0b201e1a9e2d23d9804afecb6ca681", size = 79687, upload-time = "2026-03-09T13:15:33.204Z" }, + { url = "https://files.pythonhosted.org/packages/99/a2/ca7dc962848040befed12732dff6acae7fb3c4f6fc4272b3f6c9a30b8713/kiwisolver-1.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:58f812017cd2985c21fbffb4864d59174d4903dd66fa23815e74bbc7a0e2dd57", size = 70032, upload-time = "2026-03-09T13:15:34.411Z" }, + { url = "https://files.pythonhosted.org/packages/1c/fa/2910df836372d8761bb6eff7d8bdcb1613b5c2e03f260efe7abe34d388a7/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-macosx_10_13_x86_64.whl", hash = "sha256:5ae8e62c147495b01a0f4765c878e9bfdf843412446a247e28df59936e99e797", size = 130262, upload-time = "2026-03-09T13:15:35.629Z" }, + { url = "https://files.pythonhosted.org/packages/0f/41/c5f71f9f00aabcc71fee8b7475e3f64747282580c2fe748961ba29b18385/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:f6764a4ccab3078db14a632420930f6186058750df066b8ea2a7106df91d3203", size = 138036, upload-time = "2026-03-09T13:15:36.894Z" }, + { url = "https://files.pythonhosted.org/packages/fa/06/7399a607f434119c6e1fdc8ec89a8d51ccccadf3341dee4ead6bd14caaf5/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31c13da98624f957b0fb1b5bae5383b2333c2c3f6793d9825dd5ce79b525cb7", size = 194295, upload-time = "2026-03-09T13:15:38.22Z" }, + { url = "https://files.pythonhosted.org/packages/b5/91/53255615acd2a1eaca307ede3c90eb550bae9c94581f8c00081b6b1c8f44/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:1f1489f769582498610e015a8ef2d36f28f505ab3096d0e16b4858a9ec214f57", size = 75987, upload-time = "2026-03-09T13:15:39.65Z" }, +] + +[[package]] +name = "langchain" +version = "1.3.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "langgraph" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/29/68/a6dbad9c22df4087a0f9e79ddd46226c442b30128bfeee538d5889492a73/langchain-1.3.14.tar.gz", hash = "sha256:1b6696c72ba3bbbce54d745e0180742c9f6ece8bbc59ed5a46c3e20b9a435929", size = 645181, upload-time = "2026-07-16T13:28:18.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/ec/0f942e78a621f8e3162ff1ed24284f469aaf51fb4607ee5831c626f2b2bc/langchain-1.3.14-py3-none-any.whl", hash = "sha256:4d10dbe91005952cddd56d0dc77aa108964da6bae90ab20063653957e901f782", size = 139560, upload-time = "2026-07-16T13:28:16.498Z" }, +] + +[[package]] +name = "langchain-core" +version = "1.5.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonpatch" }, + { name = "langchain-protocol" }, + { name = "langsmith" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "tenacity" }, + { name = "typing-extensions" }, + { name = "uuid-utils" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/65/3e/63af6b9d76d9be907c7c524d6ec18a2efed7e0e2d123fea0230d78dbd73f/langchain_core-1.5.3.tar.gz", hash = "sha256:a56457ac444fef41e9404443c187f0ecea708d36e816ea4ba9573c027f7d1a2d", size = 972461, upload-time = "2026-07-30T14:55:55.833Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/e6/c7c39efe0bc7e1b7c3d8f54f85846e04c901913c3d3e99068b218558c6f1/langchain_core-1.5.3-py3-none-any.whl", hash = "sha256:48b56fa580277209594dd7baf837f5b9a2a3651613f34ff9fb1728b429df015f", size = 561687, upload-time = "2026-07-30T14:55:54.419Z" }, +] + +[[package]] +name = "langchain-openai" +version = "1.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "openai" }, + { name = "tiktoken" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bf/1b/a83bf6cae4632363cef0b6f2ee1b4f62c8a5ebcf22cd8ef24430a736c2a8/langchain_openai-1.4.1.tar.gz", hash = "sha256:6d16be615d997db80294731b8e768783f1fb8e0313668e64acd50cd68acbad20", size = 3262416, upload-time = "2026-07-23T20:31:13.053Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/1c/8b604dc8be2735c8ae5c655e520066231057d1301958c20c776e62bd00fb/langchain_openai-1.4.1-py3-none-any.whl", hash = "sha256:8528bb34cc78fdfd2d895573c7917f9441cbb82db5f18ae0e6b3b75d95bdefb3", size = 122067, upload-time = "2026-07-23T20:31:11.809Z" }, +] + +[[package]] +name = "langchain-protocol" +version = "0.0.18" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d2/59/b5959aea96faa9146e2e49a7a22882b3528c62efafe9a6a95beab30c2305/langchain_protocol-0.0.18.tar.gz", hash = "sha256:ec3e11782f1ed0c9db38e5a9ed01b0e7a0d3fba406faa8aef6594b73c56a63e6", size = 6150, upload-time = "2026-06-18T17:08:26.959Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/2e/d82db9eec13ad0f72e7aaad5c4bc730ab111934fdc83c85523206eb9b0a0/langchain_protocol-0.0.18-py3-none-any.whl", hash = "sha256:70b53a86fbf9cedc863555effe44da192ab02d556ddbf2cf95b8873adcf41b5a", size = 7221, upload-time = "2026-06-18T17:08:25.996Z" }, +] + +[[package]] +name = "langgraph" +version = "1.2.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "langgraph-checkpoint" }, + { name = "langgraph-prebuilt" }, + { name = "langgraph-sdk" }, + { name = "pydantic" }, + { name = "xxhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/70/1d/a32f3caf4b3d60651656c0d64976b48d168653e81c71bb7512e9a31541aa/langgraph-1.2.10.tar.gz", hash = "sha256:05a183a746ed570a06c7c1b879920163509a75df9e44e92dd2238218d677fd37", size = 723404, upload-time = "2026-07-28T18:33:51.441Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/4d/3fc3e2535ee2c731130d71371848ebc6d4a9d2e8ae6060b11987ba134951/langgraph-1.2.10-py3-none-any.whl", hash = "sha256:52c48bd42fa31a1de0e1c0f0ebfe342e11ca2957b8b3563f83dbd60d8e30f921", size = 247753, upload-time = "2026-07-28T18:33:50.028Z" }, +] + +[[package]] +name = "langgraph-checkpoint" +version = "4.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "ormsgpack" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/47/886af6f886f0bff2273164a45f008694e48a96ff3cd25ff0228f2aa9480e/langgraph_checkpoint-4.1.1.tar.gz", hash = "sha256:6c2bdb530c91f91d7d9c1bd100925d0fc4f498d418c17f3587d1526279482a25", size = 184020, upload-time = "2026-05-22T16:57:38.503Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/b4/71425e3e38be92611300b9cc5e46a5bf98ab23f5ea8a75b73d02a2f1413c/langgraph_checkpoint-4.1.1-py3-none-any.whl", hash = "sha256:25d29144b082827218e7bc3f1e9b0566a4bb007895cd6cc26f66a8428739f56e", size = 56212, upload-time = "2026-05-22T16:57:37.203Z" }, +] + +[[package]] +name = "langgraph-prebuilt" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "langgraph-checkpoint" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/29/66/ed9b93f56bc17ef22d551892f0ac2b225a97fe0fcf23a511b857f70d590b/langgraph_prebuilt-1.1.0.tar.gz", hash = "sha256:3c579cf6eed2d17f9c157c2d0fcaddcd8688524e7022d3b22b37a3bf4589d528", size = 178833, upload-time = "2026-05-12T03:37:49.332Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/43/3fe1a700b8490ed02679cdbbc8c915eb23a092faf496c9c1118abcd10be3/langgraph_prebuilt-1.1.0-py3-none-any.whl", hash = "sha256:51e311747d755b751d5c6b39b0c1446124d3a7643d2515017e6714b323508fc9", size = 41043, upload-time = "2026-05-12T03:37:48.007Z" }, +] + +[[package]] +name = "langgraph-sdk" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "langchain-core" }, + { name = "langchain-protocol" }, + { name = "orjson" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b4/2b/bd8ac26d4e97f6df88ef05ce5b6a38945a3903e1025d926f4752aa88aa97/langgraph_sdk-0.4.2.tar.gz", hash = "sha256:b88f0f5f6328ac0680d6790614a905b2bcfa257f2276dba4e38f0e86db0aa738", size = 348327, upload-time = "2026-06-01T17:51:19.856Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/05/aac507337cceae773c2cc9ab91eb6301963af7aeeb55b4217a00e15aff17/langgraph_sdk-0.4.2-py3-none-any.whl", hash = "sha256:75fa5096c1177ce39c847096a8fe3745ffd480ddb412995f836e9f5f884c43dd", size = 160521, upload-time = "2026-06-01T17:51:18.849Z" }, +] + +[[package]] +name = "langsmith" +version = "0.10.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "orjson", marker = "platform_python_implementation != 'PyPy'" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "requests-toolbelt" }, + { name = "sniffio" }, + { name = "typing-extensions" }, + { name = "uuid-utils" }, + { name = "websockets" }, + { name = "xxhash" }, + { name = "zstandard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/99/bb/bce9faa416dfd28e1cf60bf6299e9569f9e8483b0ed22eed1d6aefc9e81c/langsmith-0.10.15.tar.gz", hash = "sha256:eefc562b29eb642a635b459e5bb44ca574380d7f32fe840acf28cd603c168647", size = 4790873, upload-time = "2026-07-31T18:15:18.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/7a/58602b770741bc84b0b35b580914f335f6663f4ea699b95eb12b074e70b8/langsmith-0.10.15-py3-none-any.whl", hash = "sha256:7afd7979a9cdf846a88c980e0a31ed518c33631d29e672adbfbb33446f3817cf", size = 731606, upload-time = "2026-07-31T18:15:16.471Z" }, +] + +[[package]] +name = "lazy-loader" +version = "0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/ac/21a1f8aa3777f5658576777ea76bfb124b702c520bbe90edf4ae9915eafa/lazy_loader-0.5.tar.gz", hash = "sha256:717f9179a0dbed357012ddad50a5ad3d5e4d9a0b8712680d4e687f5e6e6ed9b3", size = 15294, upload-time = "2026-03-06T15:45:09.054Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/a1/8d812e53a5da1687abb10445275d41a8b13adb781bbf7196ddbcf8d88505/lazy_loader-0.5-py3-none-any.whl", hash = "sha256:ab0ea149e9c554d4ffeeb21105ac60bed7f3b4fd69b1d2360a4add51b170b005", size = 8044, upload-time = "2026-03-06T15:45:07.668Z" }, +] + +[[package]] +name = "levenshtein" +version = "0.27.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "rapidfuzz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/56/dcf68853b062e3b94bdc3d011cc4198779abc5b9dc134146a062920ce2e2/levenshtein-0.27.3.tar.gz", hash = "sha256:1ac326b2c84215795163d8a5af471188918b8797b4953ec87aaba22c9c1f9fc0", size = 393269, upload-time = "2025-11-01T12:14:31.04Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/8e/3be9d8e0245704e3af5258fb6cb157c3d59902e1351e95edf6ed8a8c0434/levenshtein-0.27.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2de7f095b0ca8e44de9de986ccba661cd0dec3511c751b499e76b60da46805e9", size = 169622, upload-time = "2025-11-01T12:13:10.026Z" }, + { url = "https://files.pythonhosted.org/packages/a6/42/a2b2fda5e8caf6ecd5aac142f946a77574a3961e65da62c12fd7e48e5cb1/levenshtein-0.27.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d9b8b29e5d5145a3c958664c85151b1bb4b26e4ca764380b947e6a96a321217c", size = 159183, upload-time = "2025-11-01T12:13:11.197Z" }, + { url = "https://files.pythonhosted.org/packages/eb/c4/f083fabbd61c449752df1746533538f4a8629e8811931b52f66e6c4290ad/levenshtein-0.27.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fc975465a51b1c5889eadee1a583b81fba46372b4b22df28973e49e8ddb8f54a", size = 133120, upload-time = "2025-11-01T12:13:12.363Z" }, + { url = "https://files.pythonhosted.org/packages/4e/e5/b6421e04cb0629615b8efd6d4d167dd2b1afb5097b87bb83cd992004dcca/levenshtein-0.27.3-cp312-cp312-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:57573ed885118554770979fdee584071b66103f6d50beddeabb54607a1213d81", size = 114988, upload-time = "2025-11-01T12:13:13.486Z" }, + { url = "https://files.pythonhosted.org/packages/e5/77/39ee0e8d3028e90178e1031530ccc98563f8f2f0d905ec784669dcf0fa90/levenshtein-0.27.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23aff800a6dd5d91bb3754a6092085aa7ad46b28e497682c155c74f681cfaa2d", size = 153346, upload-time = "2025-11-01T12:13:14.744Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/c0f367bbd260dbd7a4e134fd21f459e0f5eac43deac507952b46a1d8a93a/levenshtein-0.27.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c08a952432b8ad9dccb145f812176db94c52cda732311ddc08d29fd3bf185b0a", size = 1114538, upload-time = "2025-11-01T12:13:15.851Z" }, + { url = "https://files.pythonhosted.org/packages/d8/ef/ae71433f7b4db0bd2af7974785e36cdec899919203fb82e647c5a6109c07/levenshtein-0.27.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:3bfcb2d78ab9cc06a1e75da8fcfb7a430fe513d66cfe54c07e50f32805e5e6db", size = 1009734, upload-time = "2025-11-01T12:13:17.212Z" }, + { url = "https://files.pythonhosted.org/packages/27/dc/62c28b812dcb0953fc32ab7adf3d0e814e43c8560bb28d9269a44d874adf/levenshtein-0.27.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ba7235f6dcb31a217247468295e2dd4c6c1d3ac81629dc5d355d93e1a5f4c185", size = 1185581, upload-time = "2025-11-01T12:13:18.661Z" }, + { url = "https://files.pythonhosted.org/packages/56/e8/2e7ab9c565793220edb8e5432f9a846386a157075bdd032a90e9585bce38/levenshtein-0.27.3-cp312-cp312-win32.whl", hash = "sha256:ea80d70f1d18c161a209be556b9094968627cbaae620e102459ef9c320a98cbb", size = 84660, upload-time = "2025-11-01T12:13:19.87Z" }, + { url = "https://files.pythonhosted.org/packages/2c/a6/907a1fc8587dc91c40156973e09d106ab064c06eb28dc4700ba0fe54d654/levenshtein-0.27.3-cp312-cp312-win_amd64.whl", hash = "sha256:fbaa1219d9b2d955339a37e684256a861e9274a3fe3a6ee1b8ea8724c3231ed9", size = 94909, upload-time = "2025-11-01T12:13:21.323Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d6/e04f0ddf6a71df3cdd1817b71703490ac874601ed460b2af172d3752c321/levenshtein-0.27.3-cp312-cp312-win_arm64.whl", hash = "sha256:2edbaa84f887ea1d9d8e4440af3fdda44769a7855d581c6248d7ee51518402a8", size = 87358, upload-time = "2025-11-01T12:13:22.393Z" }, + { url = "https://files.pythonhosted.org/packages/3e/f2/162e9ea7490b36bbf05776c8e3a8114c75aa78546ddda8e8f36731db3da6/levenshtein-0.27.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e55aa9f9453fd89d4a9ff1f3c4a650b307d5f61a7eed0568a52fbd2ff2eba107", size = 169230, upload-time = "2025-11-01T12:13:23.735Z" }, + { url = "https://files.pythonhosted.org/packages/01/2d/7316ba7f94e3d60e89bd120526bc71e4812866bb7162767a2a10f73f72c5/levenshtein-0.27.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ae4d484453c48939ecd01c5c213530c68dd5cd6e5090f0091ef69799ec7a8a9f", size = 158643, upload-time = "2025-11-01T12:13:25.549Z" }, + { url = "https://files.pythonhosted.org/packages/5e/87/85433cb1e51c45016f061d96fea3106b6969f700e2cbb56c15de82d0deeb/levenshtein-0.27.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d18659832567ee387b266be390da0de356a3aa6cf0e8bc009b6042d8188e131f", size = 132881, upload-time = "2025-11-01T12:13:26.822Z" }, + { url = "https://files.pythonhosted.org/packages/40/1c/3ce66c9a7da169a43dd89146d69df9dec935e6f86c70c6404f48d1291d2c/levenshtein-0.27.3-cp313-cp313-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:027b3d142cc8ea2ab4e60444d7175f65a94dde22a54382b2f7b47cc24936eb53", size = 114650, upload-time = "2025-11-01T12:13:28.382Z" }, + { url = "https://files.pythonhosted.org/packages/73/60/7138e98884ca105c76ef192f5b43165d6eac6f32b432853ebe9f09ee50c9/levenshtein-0.27.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffdca6989368cc64f347f0423c528520f12775b812e170a0eb0c10e4c9b0f3ff", size = 153127, upload-time = "2025-11-01T12:13:29.781Z" }, + { url = "https://files.pythonhosted.org/packages/df/8f/664ac8b83026d7d1382866b68babae17e92b7b6ff8dc3c6205c0066b8ce1/levenshtein-0.27.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fa00ab389386032b02a1c9050ec3c6aa824d2bbcc692548fdc44a46b71c058c6", size = 1114602, upload-time = "2025-11-01T12:13:31.651Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c8/8905d96cf2d7ed6af7eb39a8be0925ef335729473c1e9d1f56230ecaffc5/levenshtein-0.27.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:691c9003c6c481b899a5c2f72e8ce05a6d956a9668dc75f2a3ce9f4381a76dc6", size = 1008036, upload-time = "2025-11-01T12:13:33.006Z" }, + { url = "https://files.pythonhosted.org/packages/c7/57/01c37608121380a6357a297625562adad1c1fc8058d4f62279b735108927/levenshtein-0.27.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:12f7fc8bf0c24492fe97905348e020b55b9fc6dbaab7cd452566d1a466cb5e15", size = 1185338, upload-time = "2025-11-01T12:13:34.452Z" }, + { url = "https://files.pythonhosted.org/packages/dd/57/bceab41d40b58dee7927a8d1d18ed3bff7c95c5e530fb60093ce741a8c26/levenshtein-0.27.3-cp313-cp313-win32.whl", hash = "sha256:9f4872e4e19ee48eed39f214eea4eca42e5ef303f8a4a488d8312370674dbf3a", size = 84562, upload-time = "2025-11-01T12:13:35.858Z" }, + { url = "https://files.pythonhosted.org/packages/42/1d/74f1ff589bb687d0cad2bbdceef208dc070f56d1e38a3831da8c00bf13bb/levenshtein-0.27.3-cp313-cp313-win_amd64.whl", hash = "sha256:83aa2422e9a9af2c9d3e56a53e3e8de6bae58d1793628cae48c4282577c5c2c6", size = 94658, upload-time = "2025-11-01T12:13:36.963Z" }, + { url = "https://files.pythonhosted.org/packages/21/3c/22c86d3c8f254141096fd6089d2e9fdf98b1472c7a5d79d36d3557ec2d83/levenshtein-0.27.3-cp313-cp313-win_arm64.whl", hash = "sha256:d4adaf1edbcf38c3f2e290b52f4dcb5c6deff20308c26ef1127a106bc2d23e9f", size = 86929, upload-time = "2025-11-01T12:13:37.997Z" }, + { url = "https://files.pythonhosted.org/packages/0e/bc/9b7cf1b5fa098b86844d42de22549304699deff309c5c9e28b9a3fc4076a/levenshtein-0.27.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:272e24764b8210337b65a1cfd69ce40df5d2de1a3baf1234e7f06d2826ba2e7a", size = 170360, upload-time = "2025-11-01T12:13:39.019Z" }, + { url = "https://files.pythonhosted.org/packages/dc/95/997f2c83bd4712426bf0de8143b5e4403c7ebbafb5d1271983e774de3ae7/levenshtein-0.27.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:329a8e748a4e14d56daaa11f07bce3fde53385d05bad6b3f6dd9ee7802cdc915", size = 159098, upload-time = "2025-11-01T12:13:40.17Z" }, + { url = "https://files.pythonhosted.org/packages/fc/96/123c3316ae2f72c73be4fba9756924af015da4c0e5b12804f5753c0ee511/levenshtein-0.27.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a5fea1a9c6b9cc8729e467e2174b4359ff6bac27356bb5f31898e596b4ce133a", size = 136655, upload-time = "2025-11-01T12:13:41.262Z" }, + { url = "https://files.pythonhosted.org/packages/45/72/a3180d437736b1b9eacc3100be655a756deafb91de47c762d40eb45a9d91/levenshtein-0.27.3-cp313-cp313t-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3a61aa825819b6356555091d8a575d1235bd9c3753a68316a261af4856c3b487", size = 117511, upload-time = "2025-11-01T12:13:42.647Z" }, + { url = "https://files.pythonhosted.org/packages/61/f9/ba7c546a4b99347938e6661104064ab6a3651c601d59f241ffdc37510ecc/levenshtein-0.27.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a51de7a514e8183f0a82f2947d01b014d2391426543b1c076bf5a26328cec4e4", size = 155656, upload-time = "2025-11-01T12:13:44.208Z" }, + { url = "https://files.pythonhosted.org/packages/42/cd/5edd6e1e02c3e47c8121761756dd0f85f816b636f25509118b687e6b0f96/levenshtein-0.27.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:53cbf726d6e92040c9be7e594d959d496bd62597ea48eba9d96105898acbeafe", size = 1116689, upload-time = "2025-11-01T12:13:45.485Z" }, + { url = "https://files.pythonhosted.org/packages/95/67/25ca0119e0c6ec17226c72638f48ef8887124597ac48ad5da111c0b3a825/levenshtein-0.27.3-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:191b358afead8561c4fcfed22f83c13bb6c8da5f5789e277f0c5aa1c45ca612f", size = 1003166, upload-time = "2025-11-01T12:13:47.126Z" }, + { url = "https://files.pythonhosted.org/packages/45/64/ab216f3fb3cef1ee7e222665537f9340d828ef84c99409ba31f2ef2a3947/levenshtein-0.27.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ba1318d0635b834b8f0397014a7c43f007e65fce396a47614780c881bdff828b", size = 1189362, upload-time = "2025-11-01T12:13:48.627Z" }, + { url = "https://files.pythonhosted.org/packages/31/58/b150034858de0899a5a222974b6710618ebc0779a0695df070f7ab559a0b/levenshtein-0.27.3-cp313-cp313t-win32.whl", hash = "sha256:8dd9e1db6c3b35567043e155a686e4827c4aa28a594bd81e3eea84d3a1bd5875", size = 86149, upload-time = "2025-11-01T12:13:50.588Z" }, + { url = "https://files.pythonhosted.org/packages/0a/c4/bbe46a11073641450200e6a604b3b62d311166e8061c492612a40e560e85/levenshtein-0.27.3-cp313-cp313t-win_amd64.whl", hash = "sha256:7813ecdac7a6223264ebfea0c8d69959c43d21a99694ef28018d22c4265c2af6", size = 96685, upload-time = "2025-11-01T12:13:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/23/65/30b362ad9bfc1085741776a08b6ddee3f434e9daac2920daaee2e26271bf/levenshtein-0.27.3-cp313-cp313t-win_arm64.whl", hash = "sha256:8f05a0d23d13a6f802c7af595d0e43f5b9b98b6ed390cec7a35cb5d6693b882b", size = 88538, upload-time = "2025-11-01T12:13:52.757Z" }, + { url = "https://files.pythonhosted.org/packages/f3/e1/2f705da403f865a5fa3449b155738dc9c53021698fd6926253a9af03180b/levenshtein-0.27.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a6728bfae9a86002f0223576675fc7e2a6e7735da47185a1d13d1eaaa73dd4be", size = 169457, upload-time = "2025-11-01T12:13:53.778Z" }, + { url = "https://files.pythonhosted.org/packages/76/2c/bb6ef359e007fe7b6b3195b68a94f4dd3ecd1885ee337ee8fbd4df55996f/levenshtein-0.27.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8e5037c4a6f97a238e24aad6f98a1e984348b7931b1b04b6bd02bd4f8238150d", size = 158680, upload-time = "2025-11-01T12:13:55.005Z" }, + { url = "https://files.pythonhosted.org/packages/51/7b/de1999f4cf1cfebc3fbbf03a6d58498952d6560d9798af4b0a566e6b6f30/levenshtein-0.27.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c6cf5ecf9026bf24cf66ad019c6583f50058fae3e1b3c20e8812455b55d597f1", size = 133167, upload-time = "2025-11-01T12:13:56.426Z" }, + { url = "https://files.pythonhosted.org/packages/c7/da/aaa7f3a0a8ae8744b284043653652db3d7d93595517f9ed8158c03287692/levenshtein-0.27.3-cp314-cp314-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9285084bd2fc19adb47dab54ed4a71f57f78fe0d754e4a01e3c75409a25aed24", size = 114530, upload-time = "2025-11-01T12:13:57.883Z" }, + { url = "https://files.pythonhosted.org/packages/29/ce/ed422816fb30ffa3bc11597b30d5deca06b4a1388707a04215da73c65b53/levenshtein-0.27.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce3bbbe92172a08b599d79956182c6b7ab6ec8d4adbe7237417a363b968ad87b", size = 153325, upload-time = "2025-11-01T12:13:59.318Z" }, + { url = "https://files.pythonhosted.org/packages/d9/5a/a225477a0bda154f19f1c07a5e35500d631ae25dfd620b479027d79f0d4c/levenshtein-0.27.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9dac48fab9d166ca90e12fb6cf6c7c8eb9c41aacf7136584411e20f7f136f745", size = 1114956, upload-time = "2025-11-01T12:14:00.543Z" }, + { url = "https://files.pythonhosted.org/packages/ca/c4/a1be1040f3cce516a5e2be68453fd0c32ac63b2e9d31f476723fd8002c09/levenshtein-0.27.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d37a83722dc5326c93d17078e926c4732dc4f3488dc017c6839e34cd16af92b7", size = 1007610, upload-time = "2025-11-01T12:14:02.036Z" }, + { url = "https://files.pythonhosted.org/packages/86/d7/6f50e8a307e0c2befd819b481eb3a4c2eacab3dd8101982423003fac8ea3/levenshtein-0.27.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3466cb8294ce586e49dd467560a153ab8d296015c538223f149f9aefd3d9f955", size = 1185379, upload-time = "2025-11-01T12:14:03.385Z" }, + { url = "https://files.pythonhosted.org/packages/6b/e5/5d8fb1b3ebd5735f53221bf95c923066bcfc132234925820128f7eee5b47/levenshtein-0.27.3-cp314-cp314-win32.whl", hash = "sha256:c848bf2457b268672b7e9e73b44f18f49856420ac50b2564cf115a6e4ef82688", size = 86328, upload-time = "2025-11-01T12:14:04.74Z" }, + { url = "https://files.pythonhosted.org/packages/30/82/8a9ccbdb4e38bd4d516f2804999dccb8cb4bcb4e33f52851735da0c73ea7/levenshtein-0.27.3-cp314-cp314-win_amd64.whl", hash = "sha256:742633f024362a4ed6ef9d7e75d68f74b041ae738985fcf55a0e6d1d4cade438", size = 96640, upload-time = "2025-11-01T12:14:06.24Z" }, + { url = "https://files.pythonhosted.org/packages/14/86/f9d15919f59f5d92c6baa500315e1fa0143a39d811427b83c54f038267ca/levenshtein-0.27.3-cp314-cp314-win_arm64.whl", hash = "sha256:9eed6851224b19e8d588ddb8eb8a4ae3c2dcabf3d1213985f0b94a67e517b1df", size = 89689, upload-time = "2025-11-01T12:14:07.379Z" }, + { url = "https://files.pythonhosted.org/packages/ed/f6/10f44975ae6dc3047b2cd260e3d4c3a5258b8d10690a42904115de24fc51/levenshtein-0.27.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:77de69a345c76227b51a4521cd85442eb3da54c7eb6a06663a20c058fc49e683", size = 170518, upload-time = "2025-11-01T12:14:09.196Z" }, + { url = "https://files.pythonhosted.org/packages/08/07/fa294a145a0c99a814a9a807614962c1ee0f5749ca691645980462027d5d/levenshtein-0.27.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:eba2756dc1f5b962b0ff80e49abb2153d5e809cc5e7fa5e85be9410ce474795d", size = 159097, upload-time = "2025-11-01T12:14:10.404Z" }, + { url = "https://files.pythonhosted.org/packages/ae/50/24bdf37813fc30f293e53b46022b091144f4737a6a66663d2235b311bb98/levenshtein-0.27.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c8fcb498287e971d84260f67808ff1a06b3f6212d80fea75cf5155db80606ff", size = 136650, upload-time = "2025-11-01T12:14:11.579Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a9/0399c7a190b277cdea3acc801129d9d30da57c3fa79519e7b8c3f080d86c/levenshtein-0.27.3-cp314-cp314t-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f067092c67464faab13e00a5c1a80da93baca8955d4d49579861400762e35591", size = 117515, upload-time = "2025-11-01T12:14:12.877Z" }, + { url = "https://files.pythonhosted.org/packages/bf/a4/1c27533e97578b385a4b8079abe8d1ce2e514717c761efbe4bf7bbd0ac2e/levenshtein-0.27.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92415f32c68491203f2855d05eef3277d376182d014cf0859c013c89f277fbbf", size = 155711, upload-time = "2025-11-01T12:14:13.985Z" }, + { url = "https://files.pythonhosted.org/packages/50/35/bbc26638394a72b1e31a685ec251c995ee66a630c7e5c86f98770928b632/levenshtein-0.27.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef61eeaf1e0a42d7d947978d981fe4b9426b98b3dd8c1582c535f10dee044c3f", size = 1116692, upload-time = "2025-11-01T12:14:15.359Z" }, + { url = "https://files.pythonhosted.org/packages/cd/83/32fcf28b388f8dc6c36b54552b9bae289dab07d43df104893158c834cbcc/levenshtein-0.27.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:103bb2e9049d1aa0d1216dd09c1c9106ecfe7541bbdc1a0490b9357d42eec8f2", size = 1003167, upload-time = "2025-11-01T12:14:17.469Z" }, + { url = "https://files.pythonhosted.org/packages/d1/79/1fbf2877ec4b819f373a32ebe3c48a61ee810693593a6015108b0be97b78/levenshtein-0.27.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6a64ddd1986b2a4c468b09544382287315c53585eb067f6e200c337741e057ee", size = 1189417, upload-time = "2025-11-01T12:14:19.081Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/dad4e09f1f7459c64172e48e40ed2baf3aa92d38205bcbd1b4ff00853701/levenshtein-0.27.3-cp314-cp314t-win32.whl", hash = "sha256:957244f27dc284ccb030a8b77b8a00deb7eefdcd70052a4b1d96f375780ae9dc", size = 88144, upload-time = "2025-11-01T12:14:20.667Z" }, + { url = "https://files.pythonhosted.org/packages/c0/61/cd51dc8b8a382e17c559a9812734c3a9afc2dab7d36253516335ee16ae50/levenshtein-0.27.3-cp314-cp314t-win_amd64.whl", hash = "sha256:ccd7eaa6d8048c3ec07c93cfbcdefd4a3ae8c6aca3a370f2023ee69341e5f076", size = 98516, upload-time = "2025-11-01T12:14:21.786Z" }, + { url = "https://files.pythonhosted.org/packages/27/5e/3fb67e882c1fee01ebb7abc1c0a6669e5ff8acd060e93bfe7229e9ce6e4f/levenshtein-0.27.3-cp314-cp314t-win_arm64.whl", hash = "sha256:1d8520b89b7a27bb5aadbcc156715619bcbf556a8ac46ad932470945dca6e1bd", size = 91020, upload-time = "2025-11-01T12:14:22.944Z" }, +] + +[[package]] +name = "lingua-language-detector" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/c5/69636ba575cca9f507dd08ffdd4a2d084fdb193aa8e4246a5335bc077678/lingua_language_detector-2.2.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:df29270e5eef3c597e725e11eee778b7111412faab466d390d22ab1d5293bbb8", size = 170204877, upload-time = "2026-03-09T14:24:06.223Z" }, + { url = "https://files.pythonhosted.org/packages/29/05/32568a1afe29e8d2060e4ffefd9d1a67aa2e423db3ab4abbf4f604c81b39/lingua_language_detector-2.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2fe367f7c112a0445218407e259338a88af770d5c84a550c20ebe11d5053f03d", size = 172495668, upload-time = "2026-03-09T14:24:18.193Z" }, + { url = "https://files.pythonhosted.org/packages/c9/64/b6212bc0eff72d76dd04649c13452318eb2abeafc397ac597242e47e3e07/lingua_language_detector-2.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ac7453c08ab9699706a92f15480ae3d4b66761c15e1577a1ba31d1635780f3a", size = 170325432, upload-time = "2026-03-09T14:24:29.41Z" }, + { url = "https://files.pythonhosted.org/packages/44/a0/7322a0c50db8f82836ef40b14986dfcfad17bd837bfa5782562fec143bf0/lingua_language_detector-2.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63d99c7570ba09525f1702e4e4b2362f8f1f7e0a0fba93a3a53d3f322e00659d", size = 170332900, upload-time = "2026-03-09T14:24:40.088Z" }, + { url = "https://files.pythonhosted.org/packages/47/b5/e6d09c3cf08580088cc85807b1b28ef8b77d8c62d50ed56144a565205787/lingua_language_detector-2.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cd54fe6505b671c0d1e33bf0436e8e9308e8802112eb5ba6fb37d2c5459ab685", size = 170500781, upload-time = "2026-03-09T14:24:51.478Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f2/ef84cc7f57854838f9b64f1b8aae07ee56827b5538b9609acb72aa6832e5/lingua_language_detector-2.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:362fbbc21da68c778f3521f42309d1ed6f54d4bd554a5701bf165419be9cc64b", size = 170586077, upload-time = "2026-03-09T14:25:04.48Z" }, + { url = "https://files.pythonhosted.org/packages/97/48/bb581e0deda48169a11d25467d9fbe3ef4792b4d5363144bbea08caa9dd2/lingua_language_detector-2.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:98baee0c51e31d0b54a92a4795aca6ca7069de9b99dc783e3456a91abd2ff692", size = 170065705, upload-time = "2026-03-09T14:25:16.796Z" }, + { url = "https://files.pythonhosted.org/packages/45/a8/197f06b3d2da6ffb580d20e0b46181ef6d34fd750c7930ec04b322767cfb/lingua_language_detector-2.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:581bfb3405dd99863b04753812021f2554545c4c2783d0faa41af44535c759a1", size = 169977215, upload-time = "2026-03-09T14:25:31.373Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d3/b4647a233d4d8ef411519c7259c5b607b20568cb993d976319ae3f260eea/lingua_language_detector-2.2.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:d52dc5a54bb245b1d9df54620810e7b72a247f8ca4276659a9893fe415faff37", size = 170204448, upload-time = "2026-03-09T14:25:41.286Z" }, + { url = "https://files.pythonhosted.org/packages/6e/cd/248053f61de66faa866bb4eb7190af1c2e67fa363f8193444a5aee5c1706/lingua_language_detector-2.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0bb20bfe60b64012cd71f85bfdf5c79fc2e916590a9f69c3a9b01a44fbfd2244", size = 172495363, upload-time = "2026-03-09T14:25:53.585Z" }, + { url = "https://files.pythonhosted.org/packages/25/88/ad5e9b8b21f4c5eeecd5d08539bf6ec869df87a491d779b8756501db6a71/lingua_language_detector-2.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ed86c6e803a585853298623d9ee683bd08bcd15c2543c045ef059a090823fc8", size = 170326018, upload-time = "2026-03-09T14:26:04.612Z" }, + { url = "https://files.pythonhosted.org/packages/53/a5/b93c76728294e4eaf01f442fa7e9da913963d638915ce0aafd0220bc9902/lingua_language_detector-2.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4fbf936b47ef4fdd7043ebb4159d4a5f1c3648028e19d6e3c60464abc5f5e195", size = 170332278, upload-time = "2026-03-09T14:26:14.118Z" }, + { url = "https://files.pythonhosted.org/packages/21/90/7f0f4c131cd0686c0f77157545b599b5023b00fa44ffb4a1c24a4c861cb3/lingua_language_detector-2.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:126899985870ada7f9630fb984a0763741bb7fde42adfc077e6f415e49e407b5", size = 170500970, upload-time = "2026-03-09T14:26:28.07Z" }, + { url = "https://files.pythonhosted.org/packages/f4/71/24d9d151ccf35cd001d8570d22dc1d305e632eee7ff1252764be8fb081f3/lingua_language_detector-2.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c0961ec8f616897f5e91c7c3a5422d2d3aa48493954f2c425f2fca522a253916", size = 170585841, upload-time = "2026-03-09T14:26:39.904Z" }, + { url = "https://files.pythonhosted.org/packages/35/a6/e087ba2c47eb86899020915fb6bf47b0f956eda9c61cabc742bc832c1b3c/lingua_language_detector-2.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:250517a581cfa098a451299aa913e9756aee9f738b0b248259fc634eeffeb2cf", size = 170065737, upload-time = "2026-03-09T14:26:53.2Z" }, + { url = "https://files.pythonhosted.org/packages/81/e7/4ed636d7d7e4605ce170ce70a566b45f70eed79ec9cdb5c9bc821892c1cd/lingua_language_detector-2.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:9fc04412287d254982612dafe2dae2073e1feeedffbee8d4ddff4b961218cb69", size = 169977074, upload-time = "2026-03-09T14:27:04.064Z" }, + { url = "https://files.pythonhosted.org/packages/0e/53/a7f52e45e7a71c3a749cc77fbc414c8948108ff406c9059197fdc77779e8/lingua_language_detector-2.2.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:4cac0e0721425342e1b10cbddfb009a7fdc75e0a79cfd0451bffc29bee0574c1", size = 170208820, upload-time = "2026-03-09T14:27:15.253Z" }, + { url = "https://files.pythonhosted.org/packages/28/0b/3dd8a1eba4ac0da9987542849bae25344bb107e5b4a153ebe09e0c8feba3/lingua_language_detector-2.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:066b56ca4e3bd324b4c76a861ab2b747d2d8d4e6eda0a4cf06291c6c039b90f4", size = 172494828, upload-time = "2026-03-09T14:27:27.087Z" }, + { url = "https://files.pythonhosted.org/packages/a6/89/7367d0f7d3b5bcc89f47e223580ec57032dfc642f27cd2a0d06f40bda147/lingua_language_detector-2.2.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b883aa34f03cd5cde7ee606bd2c18496f15b6cbd775be0dfd38311d47d6cf551", size = 170323577, upload-time = "2026-03-09T14:27:38.702Z" }, + { url = "https://files.pythonhosted.org/packages/58/0f/6dcd9de6f5257ea736693ea92b354dac0073466a1ed32ef1f9873cc4cafe/lingua_language_detector-2.2.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83badc377b0d07f349753ec3d35cf1ad74afb3ad0dce3ee672240d437705872b", size = 170331791, upload-time = "2026-03-09T14:27:49.366Z" }, + { url = "https://files.pythonhosted.org/packages/28/42/efb8119a778f0b8df175f5f79a04a21b019c7b38058042866519953c5be1/lingua_language_detector-2.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b7ef23811c8ceacbc10a08dd2f56d71590e7ca6c50e19dfd11a1e142d101199d", size = 170499780, upload-time = "2026-03-09T14:28:04.197Z" }, + { url = "https://files.pythonhosted.org/packages/7f/89/69ea8b9de230b322ce8b60e9b95463cc4cbeed73476abd9214ab699ade73/lingua_language_detector-2.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:145a11d7b7f0c8bf666de411585f53011d530c541a2cd55c2f86b3cff499f77e", size = 170584476, upload-time = "2026-03-09T14:28:18.833Z" }, + { url = "https://files.pythonhosted.org/packages/5f/c1/2e55c62abc6653383917f9d008090820182d32b8e1f19213af1c06e16411/lingua_language_detector-2.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:3423749db1861937443141e1871a726b8d70dc6e7fe4f6584c477eef5b87fc38", size = 170064682, upload-time = "2026-03-09T14:28:31.876Z" }, + { url = "https://files.pythonhosted.org/packages/44/5e/f73a74fb19c189c4070d66e9b15f1e4a032bf5e5203fb6bb6c622e16f9c0/lingua_language_detector-2.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:0ec27bc67813372baba2e0a3df2b13cd559c64bc45c5af92f6137fe5b153a525", size = 169980726, upload-time = "2026-03-09T14:28:46.047Z" }, +] + +[[package]] +name = "loguru" +version = "0.7.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "win32-setctime", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559, upload-time = "2024-12-06T11:20:56.608Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" }, +] + +[[package]] +name = "lxml" +version = "6.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/3b/aab6728cae887456f409b4d75e8a01856e4f04bd510de38052a47768b680/lxml-6.1.1.tar.gz", hash = "sha256:ba96ae44888e0185281e937633a743ea90d5a196c6000f82565ebb0580012d40", size = 4197430, upload-time = "2026-05-18T19:19:06.424Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/6e/c4add832b6fc1e887125b96f880d7b9b70aae5248718e046b1704bcac4b9/lxml-6.1.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:104c09bda8d2a562824c0e319d0768ce26a779b7601e0931d33b09b53c392ef7", size = 8570821, upload-time = "2026-05-18T19:17:42.068Z" }, + { url = "https://files.pythonhosted.org/packages/22/00/ff3009c88e65de8011630acf8ab5a09cb2becd2aaf47fba2f3449f6224e9/lxml-6.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:25c6997a9a534e016695a0ba06b2f07945de682731ff01065b6d5a4474179da1", size = 4624252, upload-time = "2026-05-18T19:17:47.897Z" }, + { url = "https://files.pythonhosted.org/packages/42/95/bb63f0fd62e554fe078e1fb3c8fe9083c14ddc7ad7fa178d10e57e071ac7/lxml-6.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c921ba5c51e4e9f63b8b00267d06566e1f63407408a0496da2d1d0bfc819c7fc", size = 4930746, upload-time = "2026-05-18T19:18:29.637Z" }, + { url = "https://files.pythonhosted.org/packages/eb/99/0013e8d9b5960f4f041cf0b73e2f80c23eb5205b1f7bfb20203243651359/lxml-6.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:54a7f95e4de5fb94e2f9f4b9055c6ba33bf3d628fd77a1d647c5923caa2cdcdc", size = 5093723, upload-time = "2026-05-18T19:18:34.168Z" }, + { url = "https://files.pythonhosted.org/packages/29/91/317b332636bfc7bddcff828d41b3307f50043f4b237e40849c333d80fa1a/lxml-6.1.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f2ec43df44b1f76249ee0a615334f9b5b060e1c8bd90e706dad2d14d02f383", size = 5005557, upload-time = "2026-05-18T19:18:39.798Z" }, + { url = "https://files.pythonhosted.org/packages/42/2f/cc9bf06afe70f9c9093ae60855d9759da9db601ec4080f7473319666ffd7/lxml-6.1.1-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:70ef8a7e102a1508f8121aae5b0867abd663f72c14f0a9c937e6554cb4587b7b", size = 5631036, upload-time = "2026-05-18T19:18:44.858Z" }, + { url = "https://files.pythonhosted.org/packages/08/f6/af32e23e563971ffb0fb86be52bc5be5c2c118858ffc119bf6a9039b173d/lxml-6.1.1-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ebe6af670449830d6d9b752c256a983291c766a1365ba5d5460048f9e33a7818", size = 5240367, upload-time = "2026-05-18T19:18:49.217Z" }, + { url = "https://files.pythonhosted.org/packages/78/83/8555d40948b09ce86f1bd0c68a7ac31d07b1929f92cc1b074006c97ef2d2/lxml-6.1.1-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:27acc820660aaffa4f7c087f29120e12980f7779d56d8492d263170111284740", size = 5350171, upload-time = "2026-05-18T19:18:52.779Z" }, + { url = "https://files.pythonhosted.org/packages/63/75/5d92da93729b7bad783689e6496049fa40927b45bec7bf183c981de3ca70/lxml-6.1.1-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:1db753c9115ec7100d073b744d17e25e88a8f90f5c39b2f5dd878149af59671f", size = 4694874, upload-time = "2026-05-18T19:18:55.139Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b5/3aad415a9a25b822e783f15deeb4dffccf5113030f1afa2222dd929313d9/lxml-6.1.1-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c4f469aebd783bb741c2ecb2a681008fd26bfe5c16a9a72ed5467f834e810df2", size = 5244492, upload-time = "2026-05-18T19:19:01.28Z" }, + { url = "https://files.pythonhosted.org/packages/f1/a1/5fcf7eb9904b80086aa47dcf0027de07b1bb990afad2e6823144c368ae04/lxml-6.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:766b010012d59470072c1816b5b6c69f1d243e5db36ea5968e94accf430a4635", size = 5048232, upload-time = "2026-05-18T19:18:12.67Z" }, + { url = "https://files.pythonhosted.org/packages/77/74/1f601b63c7a69fcdf10fa9b148c81da8442204194f6c55509cc485c786b9/lxml-6.1.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b8d812c6011c08b8111a15e54dd990b8923692d80adf35488bee34026c35accf", size = 4777023, upload-time = "2026-05-18T19:18:15.928Z" }, + { url = "https://files.pythonhosted.org/packages/a2/b9/7a78f51aec95b1bf780d78e12705a9f6533284f8693dc5c0e6724fa53d3f/lxml-6.1.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:fe0306bd29505a9177aac19f1877174b0e7422c222a59f70b2cd41633448c3dc", size = 5645773, upload-time = "2026-05-18T19:18:23.223Z" }, + { url = "https://files.pythonhosted.org/packages/a5/6e/98a7b7ad54e4e74fa1f20fff776913980619d0ebe5558232d7da6580bdd8/lxml-6.1.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5ba186ad207446c65d3bb3d3e0412b032b1d9f595e59861e2354798c5703d955", size = 5233088, upload-time = "2026-05-18T19:18:31.433Z" }, + { url = "https://files.pythonhosted.org/packages/65/d1/bc0ed2427bf609f2ee10da303a6a226f9c8bce94f945dc29a32ce55de6e4/lxml-6.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aa366a1e55b8ebfe8ca8ddc3cfe75c8ebade181aeb0f661d0cb05986b647f72a", size = 5260995, upload-time = "2026-05-18T19:18:37.091Z" }, + { url = "https://files.pythonhosted.org/packages/69/8b/6772e1a4b513fc50a8d931f19edde0e13ae6918510a1e13ff67864f3e5ed/lxml-6.1.1-cp312-cp312-win32.whl", hash = "sha256:126c93f7f56f0eda92f6d8c619edc463a4f23d9252f1c9d0405a76f25fa9f11a", size = 3596382, upload-time = "2026-05-18T19:17:18.37Z" }, + { url = "https://files.pythonhosted.org/packages/1b/89/45198e9624762af2dfd2cb8782598477ceb29f6e59caab560388ae1f4ec1/lxml-6.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:26e6eda8d38c1fcab1090dd196ee87cbd13788e531937610e2589085de074e77", size = 3997255, upload-time = "2026-05-18T19:17:56.781Z" }, + { url = "https://files.pythonhosted.org/packages/90/a9/7a54b6834088d9ae528a7b780584ba6a39a9457b0ac330479f20ffbc9449/lxml-6.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:6540377fbd53fe1b629172288c464fb18db11ce1fa7dc15891da10aa9dcc3e7f", size = 3659610, upload-time = "2026-05-19T19:22:50.843Z" }, + { url = "https://files.pythonhosted.org/packages/a5/eb/7e6f37c5584ccbb2ff267f56fd0339016938c1c8684cfefab9b33ffc2f36/lxml-6.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:68a9198d0fc122d14bb76837de9aa80cf84caed990b5b237f532ed87d3706736", size = 8559780, upload-time = "2026-05-18T19:17:57.661Z" }, + { url = "https://files.pythonhosted.org/packages/a1/36/587c2521cf23a2cd6c9c22108aa7528f683a1f195ed7ccd23a4b1786ad36/lxml-6.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7d47866cb32fb503450b6edc9df355d10dc49836af2e89901bd6ac6b0896d9d9", size = 4618006, upload-time = "2026-05-18T19:18:04.452Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ca/ab7bfe2bf4c972af5e7878262845ead3a24a929a9b04bc11c7c1ece6c82a/lxml-6.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb7c9811bfaa8b1ed5ed319f5d370dfbcaa59d52ea64be2a5a85e18195930354", size = 4924139, upload-time = "2026-05-18T19:19:04.873Z" }, + { url = "https://files.pythonhosted.org/packages/6b/55/a0c72851dfee5ecc689f949723a73dea457758912542cb955b108eaf0d8f/lxml-6.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:762ff394d5bd56da0cf034a23dcce4e13923f15321a2adfa2ac00201dc6d3fca", size = 5082329, upload-time = "2026-05-18T19:19:09.728Z" }, + { url = "https://files.pythonhosted.org/packages/f0/b6/0608f7d61a3b96cc67e5648a3d906e31a5082093e10e7be65b3886289938/lxml-6.1.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a088f287f7d8275a33c07f2cac6c50b9319309a0200a39e7e75d80c707723099", size = 4993564, upload-time = "2026-05-18T19:19:13.608Z" }, + { url = "https://files.pythonhosted.org/packages/4c/66/ae227524b066d29d55bf0b453d93d2d793c40218657d643dcbbca13b8faf/lxml-6.1.1-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e902da4b04e6b52e5893900d4b8ab46068f75f3561f01bf1080957f9fd932ed6", size = 5613467, upload-time = "2026-05-18T19:19:16.228Z" }, + { url = "https://files.pythonhosted.org/packages/a6/76/dbe4a00b50385e40194231dcfe5a12c059de7cf90e89c83407d2b085b719/lxml-6.1.1-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d4962d4c66bf830a7e59ed6cfc17d148149898a3aefa8ec6e59763e6e3ed085", size = 5228304, upload-time = "2026-05-18T19:19:19.354Z" }, + { url = "https://files.pythonhosted.org/packages/1c/01/00b1b8442ed2041793336868ba0b9ea4b13d7da7c085c6404c207a63bf79/lxml-6.1.1-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:581d4c8ae690a6609e64862dd6b7c2489635c2d13907fc2b20f2bc200ff1d21e", size = 5341607, upload-time = "2026-05-18T19:19:22.297Z" }, + { url = "https://files.pythonhosted.org/packages/63/36/1ad29931e9a4638bb707869f01d423a6c815f82152138d1a40dfcfde2b95/lxml-6.1.1-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:876e1ff5930ed8bf295ec5ef9a8155e9b6b1876bbf1deed8b3a8069311875a8f", size = 4700168, upload-time = "2026-05-18T19:19:25.133Z" }, + { url = "https://files.pythonhosted.org/packages/3c/d1/a9536cecf9be18a0dc72d32bead283a2332d1ffebd2dd3ac70ce444686e5/lxml-6.1.1-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9eb9b5a968f6e0f6d640092a567e14529ff8cea2e29d00da6f78a79fa49f013c", size = 5232487, upload-time = "2026-05-18T19:19:28.603Z" }, + { url = "https://files.pythonhosted.org/packages/0e/77/b4fb1e03bf5d130e879214d3100092e386418807fb74dd0adc4b0a48f351/lxml-6.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:aa49e06d94aba782c6a02eecb7e507969e7e7a41b267f1b359bb35585f295d5b", size = 5044231, upload-time = "2026-05-18T19:18:42.246Z" }, + { url = "https://files.pythonhosted.org/packages/26/4c/d00daeeb0a5530c4028a9232aa1b93db3ef4ed2158c116ea73c79a9765b3/lxml-6.1.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:70cdfd80589d59e43e18005dd7244e8895e93db8ab6a620b7e23df5445a4e3d2", size = 4769450, upload-time = "2026-05-18T19:18:48.013Z" }, + { url = "https://files.pythonhosted.org/packages/ed/6a/715a3a8d156ce42f29cf014706f5410c2ff3b02267774110fc23266409fe/lxml-6.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:aad9aa39483ed8ec44d6d2e59e5b98a0d80676ef0d92f44bfc374836111f62f5", size = 5635874, upload-time = "2026-05-18T19:18:51.914Z" }, + { url = "https://files.pythonhosted.org/packages/45/37/0544bc21dde2a88f3a17b504e6fc79c0e01d25a33c2f6079724e9e72b9c7/lxml-6.1.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d49514be2f28d895c38cf9d2b72d7b9a07d00314519f456c0b50b53cfcf4c785", size = 5223987, upload-time = "2026-05-18T19:18:59.715Z" }, + { url = "https://files.pythonhosted.org/packages/4d/f8/f6a5e8185bcb28c2befae3d31f8e3df3b811cb0f47746517a81279fcafe1/lxml-6.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:47402e62c52ff5988c1e8c6c63177f5708bccf48e366dea4e3dcf1e645e04947", size = 5250276, upload-time = "2026-05-18T19:19:03.834Z" }, + { url = "https://files.pythonhosted.org/packages/c7/f2/1a2b9f1b7a49d45495369be7ef9ad05b262930f2eab3e3145706fca8083f/lxml-6.1.1-cp313-cp313-win32.whl", hash = "sha256:3483644525531e1d5762b0c44a8e18b6efba321b6dcf8a8952de10b037618bca", size = 3596903, upload-time = "2026-05-18T19:17:29.863Z" }, + { url = "https://files.pythonhosted.org/packages/e6/99/f4ffb024f238eec2131aaa09f3278fb6129cf892741bf68e1fc1afb8c100/lxml-6.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:a10bd2fd62e8ce916ececb342f348f190724a098c1faa056fdfb2a22ad5e8660", size = 3995869, upload-time = "2026-05-18T19:18:02.596Z" }, + { url = "https://files.pythonhosted.org/packages/d1/53/70eb8c5c6037f27448f1e3c54ebede9545a801ae63f0a7254afca4fe8e45/lxml-6.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:424aa57aca0897eb922aef34395bd1289b3b6f04e6bae20ea123c0c7e333cffc", size = 3658490, upload-time = "2026-05-19T19:22:53.846Z" }, + { url = "https://files.pythonhosted.org/packages/13/e2/2e325795566de01d0d7c3bb57d3c370616b2d07b01214e84eec5d3b10963/lxml-6.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:19b7ab10b210b0b3ad7985d9ac4eb66ab09a90b20fe6e2f7ba55d01a234345d0", size = 8577146, upload-time = "2026-05-18T19:18:17.765Z" }, + { url = "https://files.pythonhosted.org/packages/93/cf/5630b5e4be7d2e6bee8efe83865c925221103cf0221303b104ce134b01e2/lxml-6.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c08e5c694306507275f2290073350c4f32e383db15213b2c69e7ff39c1193840", size = 4623866, upload-time = "2026-05-18T19:18:30.669Z" }, + { url = "https://files.pythonhosted.org/packages/d2/51/3904907c063451cf8d4a5c9fe0cad95fa1f4ec57f4e3884fa0731bd7a305/lxml-6.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:74a9717fd0d82effef5c2854f0d917231d5324b5a3eb7275c43ac9fa32f97a14", size = 4950022, upload-time = "2026-05-18T19:19:31.958Z" }, + { url = "https://files.pythonhosted.org/packages/94/cd/9c7611a51c37a2830928405817cc5d56a97f64fab83cc3f628748b135749/lxml-6.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:efe0374196335f93b53269acd811b944f2e6bdc88e8894f214bd636455484909", size = 5086695, upload-time = "2026-05-18T19:19:34.764Z" }, + { url = "https://files.pythonhosted.org/packages/da/d6/24e3b5906abb0b674ff2ae195bc3ce59708df2bcd17cf17703b2d7dd643a/lxml-6.1.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac931cdc9442c1763b8a8f6cd62c0c938737eafc5be75eff88df55fc73bc0d00", size = 5031642, upload-time = "2026-05-18T19:19:37.771Z" }, + { url = "https://files.pythonhosted.org/packages/2d/db/6ec54f99019838bff54785c51da07f189eb4676861c5f2730962b0d8d665/lxml-6.1.1-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:aee395f5d0927f947758b4ec119fd5fc8ec71f07a1c5c52077b30b04c0fa6955", size = 5647338, upload-time = "2026-05-18T19:19:40.553Z" }, + { url = "https://files.pythonhosted.org/packages/42/3d/ef4dcfffd22d27a61805d8ed9f7fb888495bc6aa88648fa07c1eaa5586b6/lxml-6.1.1-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9395002973c827b3ed67db77e6ec09f092919a587022174554096a269378fb13", size = 5239528, upload-time = "2026-05-18T19:19:43.657Z" }, + { url = "https://files.pythonhosted.org/packages/62/bb/37fb3f0dff146bdcfa78eec47879273820b2a0bf350ec236ce14bd0b1c26/lxml-6.1.1-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:73bc2086f141224ebddb7fc5c6a36ca58b31b94b561e1dfe8e073e3270fad1e7", size = 5350730, upload-time = "2026-05-18T19:19:46.307Z" }, + { url = "https://files.pythonhosted.org/packages/90/42/43253f168388df4fae1f38c01df36ddb9bee39e2048167b54cdcbae85ea3/lxml-6.1.1-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3779def59032b81e44a5f70096ef6bf2082f8d901937dca354474ba09782e245", size = 4697530, upload-time = "2026-05-18T19:19:49.889Z" }, + { url = "https://files.pythonhosted.org/packages/eb/a8/c5a8504f81bbdfc8e7094c2c850cdb4ed6777fc4d5ddd9e5ab819f3b0d54/lxml-6.1.1-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:86c89b9d55ebf820ad7c90bc533410f0d098054f293351f10603c0c46ff598f5", size = 5250670, upload-time = "2026-05-18T19:19:53.199Z" }, + { url = "https://files.pythonhosted.org/packages/77/b7/c7e76ab18744d75e21f320ebf9ff9d1ceae2b54dd431ea5a64caf26c9672/lxml-6.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19607c6bbff2a44cf3fe8250abccd20942d3462473e0a721d01d379ed017e462", size = 5084485, upload-time = "2026-05-18T19:19:08.422Z" }, + { url = "https://files.pythonhosted.org/packages/31/31/b35c53f8ef7b7c31cacd23d3638652fff7bcd1deb6eedb709ab43b685908/lxml-6.1.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c6ed5141a5c7507cf3ee76bd363b0d6f801e3321adc35b5d825a23115faa5465", size = 4737635, upload-time = "2026-05-18T19:19:12.321Z" }, + { url = "https://files.pythonhosted.org/packages/d9/06/31f23c813a7fe8e0cb1b175e915b08c9bf4e86d225b210feadbdbe519667/lxml-6.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:62aeb7e85b5d60320b9d77eef2e773994e2c0ce10121b277e0a19804e1654a5a", size = 5670681, upload-time = "2026-05-18T19:19:15.001Z" }, + { url = "https://files.pythonhosted.org/packages/1a/bc/ce619bccc89b1fd9ad8a8e1330ee3f3beff9f2ff95b712d7bbcdd6e22fc3/lxml-6.1.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b1b963fd8f5caa68e99dfae060d54de1fe9cba899b8718b44a00cdca53c3e590", size = 5238229, upload-time = "2026-05-18T19:19:18.131Z" }, + { url = "https://files.pythonhosted.org/packages/2f/5d/b329acbbedc0b619ebc2be6cf7ee9ed07e80892c88d4dfd612c33805789a/lxml-6.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:63876be28efefa04a1df615b46770e82042cce445cfdce55160522f57b231ccb", size = 5264191, upload-time = "2026-05-18T19:19:21.118Z" }, + { url = "https://files.pythonhosted.org/packages/d6/85/be36fb1425b30db3c3f9df75fe86343ebffb79e6320bd7f588e25bfeac39/lxml-6.1.1-cp314-cp314-win32.whl", hash = "sha256:7f7a92e8583f06b1fd49d01158143b8461cfcd135dcb10ec807270a3051bd603", size = 3657202, upload-time = "2026-05-18T19:17:39.509Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ce/3cf9a827342269f54d405a6202397de63f07c69cbd6ce7d183a3f0cba1e9/lxml-6.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:b2d444f2e66624d68e9c6b211e28a76e22fff5fcabcfff4deac18b529b7d4137", size = 4064497, upload-time = "2026-05-18T19:18:14.662Z" }, + { url = "https://files.pythonhosted.org/packages/d9/3e/1a957bde8f0760039e627f94699f82caa782c9d838d86c3d28245ee67212/lxml-6.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:3fd9728a2735fda14f4e8235830c86b539e9661e849665bf926d3f867943b4bf", size = 3741991, upload-time = "2026-05-19T19:22:59.111Z" }, + { url = "https://files.pythonhosted.org/packages/78/b2/00ed55b3a2efa4658fb795c38d1090ec9b3e8a6c3683d4441fa517f09c3b/lxml-6.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:787b2496d0dbe8cd180984e8d29e3a6f76e7ea34db781cb3bd55e4ba1ef8b4ee", size = 8827545, upload-time = "2026-05-18T19:18:41.193Z" }, + { url = "https://files.pythonhosted.org/packages/c0/73/74573db19baa618d5f266f2407898b087ff6927115b00b71e5fc1b700847/lxml-6.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2c8daa471358dc2d6fcf02165e80ec68f77871a286df95bc5cc3816153b0fd2c", size = 4735736, upload-time = "2026-05-18T19:18:46.761Z" }, + { url = "https://files.pythonhosted.org/packages/16/02/6f7061f4f95f51e545d48e87647c54791d204a4e881be4156e7a26ba5338/lxml-6.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:acd7d70b64c0aae0c7922cca83d288a16f5f6da523637697872253415269baef", size = 4970291, upload-time = "2026-05-18T19:19:56.215Z" }, + { url = "https://files.pythonhosted.org/packages/b0/02/55fc057d8283427dea7d6edb102e7a840239c77a64a983d92f62a304c0e9/lxml-6.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4f0dd2f01f9f8a89f565d000e03abcf0a13d692a346c8d22f628d49af098777a", size = 5102822, upload-time = "2026-05-18T19:19:59.223Z" }, + { url = "https://files.pythonhosted.org/packages/e4/48/8e1cf78d89d66850121d9255a2a24414c98f775da93b90cf976956c24b14/lxml-6.1.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b7e8a14c8634bf6f7a568634cb395305a6d964aeb5b7ee32248094bed3a7e2c", size = 5027923, upload-time = "2026-05-18T19:20:01.549Z" }, + { url = "https://files.pythonhosted.org/packages/ed/00/0632a0647612c8af24d26997b3b961397daa9d5b2581444805933629a4cb/lxml-6.1.1-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:86281fbdd6a8162756f8d603f37e3435bfa38043adb79c6dc6a2dfee065e7525", size = 5595843, upload-time = "2026-05-18T19:20:03.93Z" }, + { url = "https://files.pythonhosted.org/packages/bc/86/ab008a7dc360711b66858d61c80a5979a70a09f2aa2b05d9698df80b803d/lxml-6.1.1-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5d7152ec39ca7c402d8fb9bad86140a15b9503bd0c54484e3f1bbe3dd37ceca", size = 5224515, upload-time = "2026-05-18T19:20:06.381Z" }, + { url = "https://files.pythonhosted.org/packages/75/c6/2702ff375e728e34f56d9a45339a9cf7e4427e917f542225242d63a05afa/lxml-6.1.1-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:88d8cb75b9d82858497a5393e3c63cfbf03035225e4b35a49ed7ccb151e4dc0e", size = 5312511, upload-time = "2026-05-18T19:20:09.308Z" }, + { url = "https://files.pythonhosted.org/packages/b7/57/a5807c98f87a86f10ef9ffab35516df7c0f0c4b6d5d33e9f608ab9c04a31/lxml-6.1.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f64ec5397ea6a41fc1b4af0380d79b44a755b5531dcaccd9940fb260dca93038", size = 4639206, upload-time = "2026-05-18T19:20:11.704Z" }, + { url = "https://files.pythonhosted.org/packages/1f/e1/8a0a2c35734812395f4da4eaf33748a7e5705bfb2a58b128da764339d5ec/lxml-6.1.1-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d34bbf07dbc7ca5970671b1512e928991fb5e9d95365636c9b2d8b4f53af405e", size = 5232404, upload-time = "2026-05-18T19:20:14.064Z" }, + { url = "https://files.pythonhosted.org/packages/c2/e2/0e6a4dd5ad84d01d99aa7bae7cfefd4a760a0e0f8176818241de17d9b6c0/lxml-6.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:17e0e18d4ad8adbd0399291bc44845b69d9dd68439a3cdebdf35ff902ec05072", size = 5083769, upload-time = "2026-05-18T19:19:23.758Z" }, + { url = "https://files.pythonhosted.org/packages/a0/7e/161f33d463f6ffc1c7679104b65086dea120080d49dde4d238f015aaee2f/lxml-6.1.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:3ab541146f1f6968c462d6c2ac495148e8cdba2f8347700b2141b6ec5a75bf52", size = 4758936, upload-time = "2026-05-18T19:19:27.256Z" }, + { url = "https://files.pythonhosted.org/packages/f1/fb/2369825e3f6ca99305bf9f7b7085fda91c8b0922a89e54d900974aa3ef85/lxml-6.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2a0217714657e023ef4293500f65aa20fce6164c8fd6b08fa5bd4a859fb14b9b", size = 5620296, upload-time = "2026-05-18T19:19:29.993Z" }, + { url = "https://files.pythonhosted.org/packages/30/90/d61e383146f74c5ab683947ea14dc7b82778838ab9b95ea73a23b60d0191/lxml-6.1.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:05a82eb6e1530a64f26225b55cbd178113bd0b5af1c2b625f25e5296742c26d2", size = 5228598, upload-time = "2026-05-18T19:19:33.523Z" }, + { url = "https://files.pythonhosted.org/packages/76/2d/2dafd8149e94b05bb070690efd5bb2680720681e03ff03fc57d2b70a1105/lxml-6.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9e36f163528fc50cbef305f02a5fd66d404edf7049cdaff211dbc2cba5a7013e", size = 5247845, upload-time = "2026-05-18T19:19:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/ce/68/b30e913340c380ddac9580c6e6230991fc37240ec4f64704833e4f3e2769/lxml-6.1.1-cp314-cp314t-win32.whl", hash = "sha256:649dda677cf3bd6ac9ae14007ba0c824ded8ce5808b53fc7431d9140399118c1", size = 3897345, upload-time = "2026-05-18T19:17:33.562Z" }, + { url = "https://files.pythonhosted.org/packages/3c/4e/9eb2af5335545f9fbcd7af57bcf87c6025d31eaa31b14ec184a6c8675328/lxml-6.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:793033d6c5cdf33a573f910d9bea14ef8f5771820411d118da8e1182edb53d5e", size = 4393350, upload-time = "2026-05-18T19:18:10.076Z" }, + { url = "https://files.pythonhosted.org/packages/7f/2c/0f1e93c636720e8a3eb59af2bfda99d98b55891e1c53bc30c2e0e865f01b/lxml-6.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:58bb955caba94e467d2a96da17660d2d704e0675894cba21ab8a775b8621fd1c", size = 3817223, upload-time = "2026-05-19T19:22:56.823Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + +[[package]] +name = "markdown2" +version = "2.5.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/ae/07d4a5fcaa5509221287d289323d75ac8eda5a5a4ac9de2accf7bbcc2b88/markdown2-2.5.5.tar.gz", hash = "sha256:001547e68f6e7fcf0f1cb83f7e82f48aa7d48b2c6a321f0cd20a853a8a2d1664", size = 157249, upload-time = "2026-03-02T20:46:53.411Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/af/4b3891eb0a49d6cfd5cbf3e9bf514c943afc2b0f13e2c57cc57cd88ecc21/markdown2-2.5.5-py3-none-any.whl", hash = "sha256:be798587e09d1f52d2e4d96a649c4b82a778c75f9929aad52a2c95747fa26941", size = 56250, upload-time = "2026-03-02T20:46:52.032Z" }, +] + +[[package]] +name = "markdownify" +version = "1.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/92/ab/d1297139c0e2ceb151ae564c8c4f57ac0155d8f1f8b4cbd5d6523c82ea36/markdownify-1.2.3.tar.gz", hash = "sha256:1a176f05522c8a2cb1dd3ab9d307dcdadbed5c26ae717855bfc42b3b6d38d937", size = 18852, upload-time = "2026-06-30T20:27:39.06Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/10/fa543d484e8b1199243fe20eedd02cc5af050edebce98a7293a5773df592/markdownify-1.2.3-py3-none-any.whl", hash = "sha256:a189a0bedfd14009030fde5f85bb6f77c56897cb839b5c25315dd7d4e3e290ba", size = 15732, upload-time = "2026-06-30T20:27:38.094Z" }, +] + +[[package]] +name = "matplotlib" +version = "3.11.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "contourpy" }, + { name = "cycler" }, + { name = "fonttools" }, + { name = "kiwisolver" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "pyparsing" }, + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/64/f9a391af28f518b11ad45a8a712353c94a0aefce09d3703200e5c54b610a/matplotlib-3.11.1.tar.gz", hash = "sha256:69647db5746941c793d6e445a4cd349323ffb87d9cc958c2ad84a659b4832d30", size = 32612045, upload-time = "2026-07-18T03:39:46.63Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/6c/7ef7ebcb2bd9739b2b66b18b076e077f44bb46fdbe28ca0506edb3c62c79/matplotlib-3.11.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e15ef41507f3d525f46154ac9e3ae785dacde9f20e593a25de8986267892ef74", size = 9453849, upload-time = "2026-07-18T03:38:19.593Z" }, + { url = "https://files.pythonhosted.org/packages/eb/f8/6d0c312c8d9738e7d9677f09fe5c986b3239e651a7b73a2deb38b65e4a71/matplotlib-3.11.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:21a67b961a6d597bca54fae826cd20695ba4a6e4d05424a08da6e13e3176fd6b", size = 9283113, upload-time = "2026-07-18T03:38:21.95Z" }, + { url = "https://files.pythonhosted.org/packages/c9/cf/b4ad2cc81b6672ea29ea04e64e350a9f9b493b0908ccd884c67eeff8f7b2/matplotlib-3.11.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ba8f811b8ddfac493734d6af0b2dff96919d0c28ca0d641858dab4262777c6ea", size = 10035615, upload-time = "2026-07-18T03:38:24.315Z" }, + { url = "https://files.pythonhosted.org/packages/88/90/4e10e033d9b66589d8ed98b84c95cdbb57033d57c1f41339d7393dbd2f2e/matplotlib-3.11.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c52f7ad20ef476806ed212380b1d54d20310c8b86bdc2c9a68b51f0024a44472", size = 10842559, upload-time = "2026-07-18T03:38:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/88/eb/799612d0f8cd3e816a10fec59329fca52cd2353264df80378dfc541ae855/matplotlib-3.11.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8b14eb22961fe865efb0e4ff167e333e428908b00115a8d800ccb65ee108e481", size = 10927532, upload-time = "2026-07-18T03:38:28.532Z" }, + { url = "https://files.pythonhosted.org/packages/88/89/56649bbaa2fd12e20f3be03dbcc135b0c8676d88bac17977599e3eb442a0/matplotlib-3.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:88a2a27dd9691ae448dfae4b26f59036be90c3c28757edd3553a29559d00859f", size = 9333886, upload-time = "2026-07-18T03:38:30.477Z" }, + { url = "https://files.pythonhosted.org/packages/c1/11/4d124efbbad677b7b7552f6f85a3bd432d4232f95400cea98fcd2ae36ef3/matplotlib-3.11.1-cp312-cp312-win_arm64.whl", hash = "sha256:480194afceca4df2f137c2721227d3cba67121fbf4397b69cee7f83714b0a58a", size = 9007545, upload-time = "2026-07-18T03:38:32.833Z" }, + { url = "https://files.pythonhosted.org/packages/04/6c/4798363b7fb5644e309fe1fac30216e9146c9f70859d80d588c18caf5317/matplotlib-3.11.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6771b0cd7838c6a857a7209814158c0ad09bfef878db3033dd82d70ad101f191", size = 9454341, upload-time = "2026-07-18T03:38:35.001Z" }, + { url = "https://files.pythonhosted.org/packages/59/98/6acadbe7f98df19d274bc107ac58bb439fa75df82c33dc110d71a4a8501f/matplotlib-3.11.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2abdee5ffa2fe11b2d19f7a5c63b785fb7c28cc46c7bc1814156341d9d1a33e1", size = 9283627, upload-time = "2026-07-18T03:38:37.061Z" }, + { url = "https://files.pythonhosted.org/packages/24/ea/65cec46fe241390ccea1b1754207ee28eb71c5ab866bd5f22fe47e538fa4/matplotlib-3.11.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0a19dcf73406d3746d25a5ed42d713604c9a3e024d129b102852b0d941cb9f3", size = 10035860, upload-time = "2026-07-18T03:38:39.663Z" }, + { url = "https://files.pythonhosted.org/packages/c7/10/63fdccccbabe002fb0960876baabc5e3f24d9c1bb4cfb25651457f74b3a0/matplotlib-3.11.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7389b77ed2ab0552f46d9a90b81b7b8e6dfcdc42adc36c37a0865799843e0e3e", size = 10843594, upload-time = "2026-07-18T03:38:42.144Z" }, + { url = "https://files.pythonhosted.org/packages/98/51/a1155945bff7b91381875022ac1522c5dfdac0d006be8e7df389b3134eae/matplotlib-3.11.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c90be0b73568da4f662afac580956a76e308437e641b4a45aa08925eeb67d95f", size = 10927962, upload-time = "2026-07-18T03:38:44.302Z" }, + { url = "https://files.pythonhosted.org/packages/0d/3a/3d5e1f42dc761bf53401a62a83ff93389b37de9d2c093b2a3aa49ac34f1b/matplotlib-3.11.1-cp313-cp313-win_amd64.whl", hash = "sha256:68408341f2312836fbbdf6b3c78047f65b2d8752f5fd221c3e72d348f5b34f8b", size = 9334074, upload-time = "2026-07-18T03:38:46.616Z" }, + { url = "https://files.pythonhosted.org/packages/e2/db/3f5ea5a5b64060ef5e1ff60a19170423e41ce21b8497a6fe15a36e0b43e3/matplotlib-3.11.1-cp313-cp313-win_arm64.whl", hash = "sha256:0c1f44890d435c1b4ef52f701ad5828cb450ea97bcc83918fda6be74965d6cd2", size = 9007662, upload-time = "2026-07-18T03:38:49.112Z" }, + { url = "https://files.pythonhosted.org/packages/98/6e/c7ae5e0531425b69c0826b00ebbc264c85cab853f1cd6e096c9983c2cdc1/matplotlib-3.11.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:5e510088c27a89d53580a752f959146893563e63c330e161d159b0fee652af6f", size = 9503790, upload-time = "2026-07-18T03:38:51.527Z" }, + { url = "https://files.pythonhosted.org/packages/92/79/15be162e0a2ed546939674e2e97d0e33ec2447d86d4d4e611fa295bb178c/matplotlib-3.11.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:1524e2bdd48a93557aa47ddcfe9c225dfdd57d5a01a5c49128c20f0632980ee1", size = 9336148, upload-time = "2026-07-18T03:38:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/6a/7f/36ffe144fc4aacfe0e3ed2318f72b6755d1e73b041d619b4d393e60f5a66/matplotlib-3.11.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:11664c551345553db92e61cae6cf1376f138f8c47cafdf13b64b18f3e3e9e464", size = 10049244, upload-time = "2026-07-18T03:38:55.911Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/55812d68c0a840d3a463638f48c00ab1fe338518ec49a640cb6473b444af/matplotlib-3.11.1-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e1f8922ba31959cf6a9dfb51be64b7f7bc582801a3957dc0c2f3afcd3537adf", size = 10860798, upload-time = "2026-07-18T03:38:58.282Z" }, + { url = "https://files.pythonhosted.org/packages/7a/64/cca444b4eb5e6c768c44fc5e1f0b5211f20ca2b282778051996e996a2bdf/matplotlib-3.11.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:83235693abde86e5e0129998f80ee39fc7f58e6d56a88fafb28a9278833e9d5f", size = 10943282, upload-time = "2026-07-18T03:39:00.465Z" }, + { url = "https://files.pythonhosted.org/packages/e5/0f/a49c329d394f2e9ef38506982107e8b04ecf94dd41a9d8423ff82cc737c7/matplotlib-3.11.1-cp313-cp313t-win_amd64.whl", hash = "sha256:9a076f4fc5cdc43fdf510f5981418d25c2db4973418d9f22d8bb3dc8045ada78", size = 9383532, upload-time = "2026-07-18T03:39:02.468Z" }, + { url = "https://files.pythonhosted.org/packages/e4/50/103e86afb806d8f64d04ede14e4cfc09dbfc25f512421ff85fdd6ebd59cf/matplotlib-3.11.1-cp313-cp313t-win_arm64.whl", hash = "sha256:216fbb93a74add02ddb4cb38ef5348f59ac00b3e84567eaf16598772d40e150a", size = 9059665, upload-time = "2026-07-18T03:39:04.607Z" }, + { url = "https://files.pythonhosted.org/packages/35/04/3079499fa8cb661ea66d13d6439d5a3ae6710a7afd5c7f72e08914f275f8/matplotlib-3.11.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:30c492d4ba9448595b6fd8708c6725963f8148e25c0d8842948da5b05f0ee8d3", size = 9456022, upload-time = "2026-07-18T03:39:07.041Z" }, + { url = "https://files.pythonhosted.org/packages/53/a2/69acfe84ec1f32930e801a5782a07fc5c79c8c6599a507b806d859d5da8e/matplotlib-3.11.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ac104be2768ffdd8655db9e71b768cbb45f2b9aa7b450cf1595e8f65d3822319", size = 9285475, upload-time = "2026-07-18T03:39:09.562Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b3/31b15a2ca56d4ddd6aaa1c884c2f51cf9a61cfaf5ca6f6fbd6343d38e6df/matplotlib-3.11.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6be943cb68bc6660ead58c55b3aa6366cba2ef7feb06460fbcce32360376f19f", size = 10847102, upload-time = "2026-07-18T03:39:11.532Z" }, + { url = "https://files.pythonhosted.org/packages/64/0d/a17e966e620545c1548125af0b29ac812dd17b197a18a7462ac12fa859ee/matplotlib-3.11.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5af0dcda57d471440a7b5b623e70e0a61003518443d9098f211a96ecfbbc25be", size = 11131087, upload-time = "2026-07-18T03:39:13.764Z" }, + { url = "https://files.pythonhosted.org/packages/97/c5/5e100efdd67abb7de20befaa333612ef9bfc63417fb71398f904f25d083c/matplotlib-3.11.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3d3fd84082b1afbd9398466c81309e20045be20d48fe0fb18c43504d164cbbb2", size = 10929036, upload-time = "2026-07-18T03:39:16.888Z" }, + { url = "https://files.pythonhosted.org/packages/ce/04/d719a0a36930ecc8dfc801ff340f9dcfc4223f8ca5d39d06b4020032fff8/matplotlib-3.11.1-cp314-cp314-win_amd64.whl", hash = "sha256:9601a1e90be21e4884c53b4f3dc3ee0544654946f9975258d691f1c2e2f119c6", size = 9489571, upload-time = "2026-07-18T03:39:19.449Z" }, + { url = "https://files.pythonhosted.org/packages/48/65/facabdc2f1f6caba7e856db64dfedddca25f7608df07d96a1c8fd114fd3b/matplotlib-3.11.1-cp314-cp314-win_arm64.whl", hash = "sha256:ae30c6109848ac0f9fa36c5d6270938487614c47ba31860bd5361266dabc5685", size = 9164486, upload-time = "2026-07-18T03:39:21.424Z" }, + { url = "https://files.pythonhosted.org/packages/88/dd/18da6cd01cf96354534f98c468a25380c68ce582a2c9dd0cae12b04af4f2/matplotlib-3.11.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:dadfe80797174e2984aae3be0b77594a3c72d2c0a40fbd4a0de48d2728caf3ae", size = 9504876, upload-time = "2026-07-18T03:39:23.633Z" }, + { url = "https://files.pythonhosted.org/packages/79/b0/f0b63555a18b79d038c81fd6126f35fc4dfce0eaff48d96103348c7cf935/matplotlib-3.11.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:89b193b255f4f6f7948dbcee3691f4f341ab05d9a8874a67b45ddb4182922eda", size = 9336120, upload-time = "2026-07-18T03:39:25.797Z" }, + { url = "https://files.pythonhosted.org/packages/c6/dd/f210ec7c4a6f198d5567237048a93d0811fb5a1f1691f13320e592f95b41/matplotlib-3.11.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:191163532cdefcb1571ca38a6d7e6474baccde64495783e6ba47aa07ec4b9bbb", size = 10858033, upload-time = "2026-07-18T03:39:27.999Z" }, + { url = "https://files.pythonhosted.org/packages/ec/d2/d6d5324507c5fbb316db48e258c09c2807f3de03d9af47017e120070926f/matplotlib-3.11.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9fdf1c818ab05d0e74002091ddaf414478a3a449ec9d51c8976d45be7e3a01e2", size = 11141827, upload-time = "2026-07-18T03:39:30.092Z" }, + { url = "https://files.pythonhosted.org/packages/0f/68/3c22e9320bdce2c4d2f1320643ef706db7a24cb7420eea28b97a2d67f5a8/matplotlib-3.11.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b937b9dba5f5f6c1e31c47abe2186c865c0914fd18f2ce0dfc39c9adcef5951d", size = 10943061, upload-time = "2026-07-18T03:39:32.356Z" }, + { url = "https://files.pythonhosted.org/packages/f6/4a/907ed190ee81a9df581e0ed5456134fc0f7cb55ffcfda2f9e54ca900761c/matplotlib-3.11.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f2912f647f3fbe1ccf085f91e213936f9101bead81a5e670565b1f1b3712f4fb", size = 9540074, upload-time = "2026-07-18T03:39:34.789Z" }, + { url = "https://files.pythonhosted.org/packages/23/d4/97c19b77e0a6e3b48581185bb65088f431cd20186076cc0f650a1757ea46/matplotlib-3.11.1-cp314-cp314t-win_arm64.whl", hash = "sha256:54d47b8ae8b579633a3902ca5b4ad6c1e132a5626d64447b2e22a66394e79987", size = 9213472, upload-time = "2026-07-18T03:39:37.141Z" }, +] + +[[package]] +name = "matplotlib-inline" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/c0/9f7c9a46090390368a4d7bcb76bb87a4a36c421e4c0792cdb53486ffac7a/matplotlib_inline-0.2.2.tar.gz", hash = "sha256:72f3fe8fce36b70d4a5b612f899090cd0401deddc4ea90e1572b9f4bfb058c79", size = 8150, upload-time = "2026-05-08T17:33:33.49Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "mmeval" +version = "0.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "opencv-python" }, + { name = "plum-dispatch" }, + { name = "pyyaml" }, + { name = "rich" }, + { name = "scipy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/5d/3703e5eeae80f0a007aefe95f58cacce5fd50658fcbbad9af5258e3cb49c/mmeval-0.2.1.tar.gz", hash = "sha256:5fa8933b7a06a4507928cfe9e171232ccbc88768025609da5c462a9c982b44e2", size = 137842, upload-time = "2023-04-03T08:07:40.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/2a/89546ea52c77efa5f7464d5e6e93fec8be2f64b8253a52c607d180e1c845/mmeval-0.2.1-py3-none-any.whl", hash = "sha256:d9b4bc08438ea91dc1859eed624697e362b12e9e8f0fb4a752f53d94b51be955", size = 189744, upload-time = "2023-04-03T08:07:38.483Z" }, +] + +[[package]] +name = "multidict" +version = "6.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/9c/f20e0e2cf80e4b2e4b1c365bf5fe104ee633c751a724246262db8f1a0b13/multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172", size = 76893, upload-time = "2026-01-26T02:43:52.754Z" }, + { url = "https://files.pythonhosted.org/packages/fe/cf/18ef143a81610136d3da8193da9d80bfe1cb548a1e2d1c775f26b23d024a/multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd", size = 45456, upload-time = "2026-01-26T02:43:53.893Z" }, + { url = "https://files.pythonhosted.org/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7", size = 43872, upload-time = "2026-01-26T02:43:55.041Z" }, + { url = "https://files.pythonhosted.org/packages/cf/3b/d6bd75dc4f3ff7c73766e04e705b00ed6dbbaccf670d9e05a12b006f5a21/multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53", size = 251018, upload-time = "2026-01-26T02:43:56.198Z" }, + { url = "https://files.pythonhosted.org/packages/fd/80/c959c5933adedb9ac15152e4067c702a808ea183a8b64cf8f31af8ad3155/multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75", size = 258883, upload-time = "2026-01-26T02:43:57.499Z" }, + { url = "https://files.pythonhosted.org/packages/86/85/7ed40adafea3d4f1c8b916e3b5cc3a8e07dfcdcb9cd72800f4ed3ca1b387/multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b", size = 242413, upload-time = "2026-01-26T02:43:58.755Z" }, + { url = "https://files.pythonhosted.org/packages/d2/57/b8565ff533e48595503c785f8361ff9a4fde4d67de25c207cd0ba3befd03/multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733", size = 268404, upload-time = "2026-01-26T02:44:00.216Z" }, + { url = "https://files.pythonhosted.org/packages/e0/50/9810c5c29350f7258180dfdcb2e52783a0632862eb334c4896ac717cebcb/multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a", size = 269456, upload-time = "2026-01-26T02:44:02.202Z" }, + { url = "https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961", size = 256322, upload-time = "2026-01-26T02:44:03.56Z" }, + { url = "https://files.pythonhosted.org/packages/31/6e/d8a26d81ac166a5592782d208dd90dfdc0a7a218adaa52b45a672b46c122/multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582", size = 253955, upload-time = "2026-01-26T02:44:04.845Z" }, + { url = "https://files.pythonhosted.org/packages/59/4c/7c672c8aad41534ba619bcd4ade7a0dc87ed6b8b5c06149b85d3dd03f0cd/multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e", size = 251254, upload-time = "2026-01-26T02:44:06.133Z" }, + { url = "https://files.pythonhosted.org/packages/7b/bd/84c24de512cbafbdbc39439f74e967f19570ce7924e3007174a29c348916/multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3", size = 252059, upload-time = "2026-01-26T02:44:07.518Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ba/f5449385510825b73d01c2d4087bf6d2fccc20a2d42ac34df93191d3dd03/multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6", size = 263588, upload-time = "2026-01-26T02:44:09.382Z" }, + { url = "https://files.pythonhosted.org/packages/d7/11/afc7c677f68f75c84a69fe37184f0f82fce13ce4b92f49f3db280b7e92b3/multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a", size = 259642, upload-time = "2026-01-26T02:44:10.73Z" }, + { url = "https://files.pythonhosted.org/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba", size = 251377, upload-time = "2026-01-26T02:44:12.042Z" }, + { url = "https://files.pythonhosted.org/packages/ca/a4/840f5b97339e27846c46307f2530a2805d9d537d8b8bd416af031cad7fa0/multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511", size = 41887, upload-time = "2026-01-26T02:44:14.245Z" }, + { url = "https://files.pythonhosted.org/packages/80/31/0b2517913687895f5904325c2069d6a3b78f66cc641a86a2baf75a05dcbb/multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19", size = 46053, upload-time = "2026-01-26T02:44:15.371Z" }, + { url = "https://files.pythonhosted.org/packages/0c/5b/aba28e4ee4006ae4c7df8d327d31025d760ffa992ea23812a601d226e682/multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf", size = 43307, upload-time = "2026-01-26T02:44:16.852Z" }, + { url = "https://files.pythonhosted.org/packages/f2/22/929c141d6c0dba87d3e1d38fbdf1ba8baba86b7776469f2bc2d3227a1e67/multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23", size = 76174, upload-time = "2026-01-26T02:44:18.509Z" }, + { url = "https://files.pythonhosted.org/packages/c7/75/bc704ae15fee974f8fccd871305e254754167dce5f9e42d88a2def741a1d/multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2", size = 45116, upload-time = "2026-01-26T02:44:19.745Z" }, + { url = "https://files.pythonhosted.org/packages/79/76/55cd7186f498ed080a18440c9013011eb548f77ae1b297206d030eb1180a/multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445", size = 43524, upload-time = "2026-01-26T02:44:21.571Z" }, + { url = "https://files.pythonhosted.org/packages/e9/3c/414842ef8d5a1628d68edee29ba0e5bcf235dbfb3ccd3ea303a7fe8c72ff/multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177", size = 249368, upload-time = "2026-01-26T02:44:22.803Z" }, + { url = "https://files.pythonhosted.org/packages/f6/32/befed7f74c458b4a525e60519fe8d87eef72bb1e99924fa2b0f9d97a221e/multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23", size = 256952, upload-time = "2026-01-26T02:44:24.306Z" }, + { url = "https://files.pythonhosted.org/packages/03/d6/c878a44ba877f366630c860fdf74bfb203c33778f12b6ac274936853c451/multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060", size = 240317, upload-time = "2026-01-26T02:44:25.772Z" }, + { url = "https://files.pythonhosted.org/packages/68/49/57421b4d7ad2e9e60e25922b08ceb37e077b90444bde6ead629095327a6f/multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d", size = 267132, upload-time = "2026-01-26T02:44:27.648Z" }, + { url = "https://files.pythonhosted.org/packages/b7/fe/ec0edd52ddbcea2a2e89e174f0206444a61440b40f39704e64dc807a70bd/multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed", size = 268140, upload-time = "2026-01-26T02:44:29.588Z" }, + { url = "https://files.pythonhosted.org/packages/b0/73/6e1b01cbeb458807aa0831742232dbdd1fa92bfa33f52a3f176b4ff3dc11/multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429", size = 254277, upload-time = "2026-01-26T02:44:30.902Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b2/5fb8c124d7561a4974c342bc8c778b471ebbeb3cc17df696f034a7e9afe7/multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6", size = 252291, upload-time = "2026-01-26T02:44:32.31Z" }, + { url = "https://files.pythonhosted.org/packages/5a/96/51d4e4e06bcce92577fcd488e22600bd38e4fd59c20cb49434d054903bd2/multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9", size = 250156, upload-time = "2026-01-26T02:44:33.734Z" }, + { url = "https://files.pythonhosted.org/packages/db/6b/420e173eec5fba721a50e2a9f89eda89d9c98fded1124f8d5c675f7a0c0f/multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c", size = 249742, upload-time = "2026-01-26T02:44:35.222Z" }, + { url = "https://files.pythonhosted.org/packages/44/a3/ec5b5bd98f306bc2aa297b8c6f11a46714a56b1e6ef5ebda50a4f5d7c5fb/multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84", size = 262221, upload-time = "2026-01-26T02:44:36.604Z" }, + { url = "https://files.pythonhosted.org/packages/cd/f7/e8c0d0da0cd1e28d10e624604e1a36bcc3353aaebdfdc3a43c72bc683a12/multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d", size = 258664, upload-time = "2026-01-26T02:44:38.008Z" }, + { url = "https://files.pythonhosted.org/packages/52/da/151a44e8016dd33feed44f730bd856a66257c1ee7aed4f44b649fb7edeb3/multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33", size = 249490, upload-time = "2026-01-26T02:44:39.386Z" }, + { url = "https://files.pythonhosted.org/packages/87/af/a3b86bf9630b732897f6fc3f4c4714b90aa4361983ccbdcd6c0339b21b0c/multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3", size = 41695, upload-time = "2026-01-26T02:44:41.318Z" }, + { url = "https://files.pythonhosted.org/packages/b2/35/e994121b0e90e46134673422dd564623f93304614f5d11886b1b3e06f503/multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5", size = 45884, upload-time = "2026-01-26T02:44:42.488Z" }, + { url = "https://files.pythonhosted.org/packages/ca/61/42d3e5dbf661242a69c97ea363f2d7b46c567da8eadef8890022be6e2ab0/multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df", size = 43122, upload-time = "2026-01-26T02:44:43.664Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b3/e6b21c6c4f314bb956016b0b3ef2162590a529b84cb831c257519e7fde44/multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1", size = 83175, upload-time = "2026-01-26T02:44:44.894Z" }, + { url = "https://files.pythonhosted.org/packages/fb/76/23ecd2abfe0957b234f6c960f4ade497f55f2c16aeb684d4ecdbf1c95791/multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963", size = 48460, upload-time = "2026-01-26T02:44:46.106Z" }, + { url = "https://files.pythonhosted.org/packages/c4/57/a0ed92b23f3a042c36bc4227b72b97eca803f5f1801c1ab77c8a212d455e/multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34", size = 46930, upload-time = "2026-01-26T02:44:47.278Z" }, + { url = "https://files.pythonhosted.org/packages/b5/66/02ec7ace29162e447f6382c495dc95826bf931d3818799bbef11e8f7df1a/multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65", size = 242582, upload-time = "2026-01-26T02:44:48.604Z" }, + { url = "https://files.pythonhosted.org/packages/58/18/64f5a795e7677670e872673aca234162514696274597b3708b2c0d276cce/multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292", size = 250031, upload-time = "2026-01-26T02:44:50.544Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ed/e192291dbbe51a8290c5686f482084d31bcd9d09af24f63358c3d42fd284/multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43", size = 228596, upload-time = "2026-01-26T02:44:51.951Z" }, + { url = "https://files.pythonhosted.org/packages/1e/7e/3562a15a60cf747397e7f2180b0a11dc0c38d9175a650e75fa1b4d325e15/multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca", size = 257492, upload-time = "2026-01-26T02:44:53.902Z" }, + { url = "https://files.pythonhosted.org/packages/24/02/7d0f9eae92b5249bb50ac1595b295f10e263dd0078ebb55115c31e0eaccd/multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd", size = 255899, upload-time = "2026-01-26T02:44:55.316Z" }, + { url = "https://files.pythonhosted.org/packages/00/e3/9b60ed9e23e64c73a5cde95269ef1330678e9c6e34dd4eb6b431b85b5a10/multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7", size = 247970, upload-time = "2026-01-26T02:44:56.783Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/538e58a63ed5cfb0bd4517e346b91da32fde409d839720f664e9a4ae4f9d/multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3", size = 245060, upload-time = "2026-01-26T02:44:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2f/d743a3045a97c895d401e9bd29aaa09b94f5cbdf1bd561609e5a6c431c70/multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4", size = 235888, upload-time = "2026-01-26T02:44:59.57Z" }, + { url = "https://files.pythonhosted.org/packages/38/83/5a325cac191ab28b63c52f14f1131f3b0a55ba3b9aa65a6d0bf2a9b921a0/multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8", size = 243554, upload-time = "2026-01-26T02:45:01.054Z" }, + { url = "https://files.pythonhosted.org/packages/20/1f/9d2327086bd15da2725ef6aae624208e2ef828ed99892b17f60c344e57ed/multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c", size = 252341, upload-time = "2026-01-26T02:45:02.484Z" }, + { url = "https://files.pythonhosted.org/packages/e8/2c/2a1aa0280cf579d0f6eed8ee5211c4f1730bd7e06c636ba2ee6aafda302e/multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52", size = 246391, upload-time = "2026-01-26T02:45:03.862Z" }, + { url = "https://files.pythonhosted.org/packages/e5/03/7ca022ffc36c5a3f6e03b179a5ceb829be9da5783e6fe395f347c0794680/multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108", size = 243422, upload-time = "2026-01-26T02:45:05.296Z" }, + { url = "https://files.pythonhosted.org/packages/dc/1d/b31650eab6c5778aceed46ba735bd97f7c7d2f54b319fa916c0f96e7805b/multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32", size = 47770, upload-time = "2026-01-26T02:45:06.754Z" }, + { url = "https://files.pythonhosted.org/packages/ac/5b/2d2d1d522e51285bd61b1e20df8f47ae1a9d80839db0b24ea783b3832832/multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8", size = 53109, upload-time = "2026-01-26T02:45:08.044Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a3/cc409ba012c83ca024a308516703cf339bdc4b696195644a7215a5164a24/multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118", size = 45573, upload-time = "2026-01-26T02:45:09.349Z" }, + { url = "https://files.pythonhosted.org/packages/91/cc/db74228a8be41884a567e88a62fd589a913708fcf180d029898c17a9a371/multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee", size = 75190, upload-time = "2026-01-26T02:45:10.651Z" }, + { url = "https://files.pythonhosted.org/packages/d5/22/492f2246bb5b534abd44804292e81eeaf835388901f0c574bac4eeec73c5/multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2", size = 44486, upload-time = "2026-01-26T02:45:11.938Z" }, + { url = "https://files.pythonhosted.org/packages/f1/4f/733c48f270565d78b4544f2baddc2fb2a245e5a8640254b12c36ac7ac68e/multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1", size = 43219, upload-time = "2026-01-26T02:45:14.346Z" }, + { url = "https://files.pythonhosted.org/packages/24/bb/2c0c2287963f4259c85e8bcbba9182ced8d7fca65c780c38e99e61629d11/multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d", size = 245132, upload-time = "2026-01-26T02:45:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/a7/f9/44d4b3064c65079d2467888794dea218d1601898ac50222ab8a9a8094460/multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31", size = 252420, upload-time = "2026-01-26T02:45:17.293Z" }, + { url = "https://files.pythonhosted.org/packages/8b/13/78f7275e73fa17b24c9a51b0bd9d73ba64bb32d0ed51b02a746eb876abe7/multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048", size = 233510, upload-time = "2026-01-26T02:45:19.356Z" }, + { url = "https://files.pythonhosted.org/packages/4b/25/8167187f62ae3cbd52da7893f58cb036b47ea3fb67138787c76800158982/multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362", size = 264094, upload-time = "2026-01-26T02:45:20.834Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e7/69a3a83b7b030cf283fb06ce074a05a02322359783424d7edf0f15fe5022/multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37", size = 260786, upload-time = "2026-01-26T02:45:22.818Z" }, + { url = "https://files.pythonhosted.org/packages/fe/3b/8ec5074bcfc450fe84273713b4b0a0dd47c0249358f5d82eb8104ffe2520/multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709", size = 248483, upload-time = "2026-01-26T02:45:24.368Z" }, + { url = "https://files.pythonhosted.org/packages/48/5a/d5a99e3acbca0e29c5d9cba8f92ceb15dce78bab963b308ae692981e3a5d/multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0", size = 248403, upload-time = "2026-01-26T02:45:25.982Z" }, + { url = "https://files.pythonhosted.org/packages/35/48/e58cd31f6c7d5102f2a4bf89f96b9cf7e00b6c6f3d04ecc44417c00a5a3c/multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb", size = 240315, upload-time = "2026-01-26T02:45:27.487Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/1cd210229559cb90b6786c30676bb0c58249ff42f942765f88793b41fdce/multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd", size = 245528, upload-time = "2026-01-26T02:45:28.991Z" }, + { url = "https://files.pythonhosted.org/packages/64/f2/6e1107d226278c876c783056b7db43d800bb64c6131cec9c8dfb6903698e/multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601", size = 258784, upload-time = "2026-01-26T02:45:30.503Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c1/11f664f14d525e4a1b5327a82d4de61a1db604ab34c6603bb3c2cc63ad34/multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1", size = 251980, upload-time = "2026-01-26T02:45:32.603Z" }, + { url = "https://files.pythonhosted.org/packages/e1/9f/75a9ac888121d0c5bbd4ecf4eead45668b1766f6baabfb3b7f66a410e231/multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b", size = 243602, upload-time = "2026-01-26T02:45:34.043Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e7/50bf7b004cc8525d80dbbbedfdc7aed3e4c323810890be4413e589074032/multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d", size = 40930, upload-time = "2026-01-26T02:45:36.278Z" }, + { url = "https://files.pythonhosted.org/packages/e0/bf/52f25716bbe93745595800f36fb17b73711f14da59ed0bb2eba141bc9f0f/multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f", size = 45074, upload-time = "2026-01-26T02:45:37.546Z" }, + { url = "https://files.pythonhosted.org/packages/97/ab/22803b03285fa3a525f48217963da3a65ae40f6a1b6f6cf2768879e208f9/multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5", size = 42471, upload-time = "2026-01-26T02:45:38.889Z" }, + { url = "https://files.pythonhosted.org/packages/e0/6d/f9293baa6146ba9507e360ea0292b6422b016907c393e2f63fc40ab7b7b5/multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581", size = 82401, upload-time = "2026-01-26T02:45:40.254Z" }, + { url = "https://files.pythonhosted.org/packages/7a/68/53b5494738d83558d87c3c71a486504d8373421c3e0dbb6d0db48ad42ee0/multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a", size = 48143, upload-time = "2026-01-26T02:45:41.635Z" }, + { url = "https://files.pythonhosted.org/packages/37/e8/5284c53310dcdc99ce5d66563f6e5773531a9b9fe9ec7a615e9bc306b05f/multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c", size = 46507, upload-time = "2026-01-26T02:45:42.99Z" }, + { url = "https://files.pythonhosted.org/packages/e4/fc/6800d0e5b3875568b4083ecf5f310dcf91d86d52573160834fb4bfcf5e4f/multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262", size = 239358, upload-time = "2026-01-26T02:45:44.376Z" }, + { url = "https://files.pythonhosted.org/packages/41/75/4ad0973179361cdf3a113905e6e088173198349131be2b390f9fa4da5fc6/multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59", size = 246884, upload-time = "2026-01-26T02:45:47.167Z" }, + { url = "https://files.pythonhosted.org/packages/c3/9c/095bb28b5da139bd41fb9a5d5caff412584f377914bd8787c2aa98717130/multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889", size = 225878, upload-time = "2026-01-26T02:45:48.698Z" }, + { url = "https://files.pythonhosted.org/packages/07/d0/c0a72000243756e8f5a277b6b514fa005f2c73d481b7d9e47cd4568aa2e4/multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4", size = 253542, upload-time = "2026-01-26T02:45:50.164Z" }, + { url = "https://files.pythonhosted.org/packages/c0/6b/f69da15289e384ecf2a68837ec8b5ad8c33e973aa18b266f50fe55f24b8c/multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d", size = 252403, upload-time = "2026-01-26T02:45:51.779Z" }, + { url = "https://files.pythonhosted.org/packages/a2/76/b9669547afa5a1a25cd93eaca91c0da1c095b06b6d2d8ec25b713588d3a1/multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609", size = 244889, upload-time = "2026-01-26T02:45:53.27Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a9/a50d2669e506dad33cfc45b5d574a205587b7b8a5f426f2fbb2e90882588/multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489", size = 241982, upload-time = "2026-01-26T02:45:54.919Z" }, + { url = "https://files.pythonhosted.org/packages/c5/bb/1609558ad8b456b4827d3c5a5b775c93b87878fd3117ed3db3423dfbce1b/multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c", size = 232415, upload-time = "2026-01-26T02:45:56.981Z" }, + { url = "https://files.pythonhosted.org/packages/d8/59/6f61039d2aa9261871e03ab9dc058a550d240f25859b05b67fd70f80d4b3/multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e", size = 240337, upload-time = "2026-01-26T02:45:58.698Z" }, + { url = "https://files.pythonhosted.org/packages/a1/29/fdc6a43c203890dc2ae9249971ecd0c41deaedfe00d25cb6564b2edd99eb/multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c", size = 248788, upload-time = "2026-01-26T02:46:00.862Z" }, + { url = "https://files.pythonhosted.org/packages/a9/14/a153a06101323e4cf086ecee3faadba52ff71633d471f9685c42e3736163/multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9", size = 242842, upload-time = "2026-01-26T02:46:02.824Z" }, + { url = "https://files.pythonhosted.org/packages/41/5f/604ae839e64a4a6efc80db94465348d3b328ee955e37acb24badbcd24d83/multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2", size = 240237, upload-time = "2026-01-26T02:46:05.898Z" }, + { url = "https://files.pythonhosted.org/packages/5f/60/c3a5187bf66f6fb546ff4ab8fb5a077cbdd832d7b1908d4365c7f74a1917/multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7", size = 48008, upload-time = "2026-01-26T02:46:07.468Z" }, + { url = "https://files.pythonhosted.org/packages/0c/f7/addf1087b860ac60e6f382240f64fb99f8bfb532bb06f7c542b83c29ca61/multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5", size = 53542, upload-time = "2026-01-26T02:46:08.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/81/4629d0aa32302ef7b2ec65c75a728cc5ff4fa410c50096174c1632e70b3e/multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2", size = 44719, upload-time = "2026-01-26T02:46:11.146Z" }, + { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, +] + +[[package]] +name = "multiprocess" +version = "0.70.19" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a2/f2/e783ac7f2aeeed14e9e12801f22529cc7e6b7ab80928d6dcce4e9f00922d/multiprocess-0.70.19.tar.gz", hash = "sha256:952021e0e6c55a4a9fe4cd787895b86e239a40e76802a789d6305398d3975897", size = 2079989, upload-time = "2026-01-19T06:47:39.744Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/45/8004d1e6b9185c1a444d6b55ac5682acf9d98035e54386d967366035a03a/multiprocess-0.70.19-py310-none-any.whl", hash = "sha256:97404393419dcb2a8385910864eedf47a3cadf82c66345b44f036420eb0b5d87", size = 134948, upload-time = "2026-01-19T06:47:32.325Z" }, + { url = "https://files.pythonhosted.org/packages/86/c2/dec9722dc3474c164a0b6bcd9a7ed7da542c98af8cabce05374abab35edd/multiprocess-0.70.19-py311-none-any.whl", hash = "sha256:928851ae7973aea4ce0eaf330bbdafb2e01398a91518d5c8818802845564f45c", size = 144457, upload-time = "2026-01-19T06:47:33.711Z" }, + { url = "https://files.pythonhosted.org/packages/71/70/38998b950a97ea279e6bd657575d22d1a2047256caf707d9a10fbce4f065/multiprocess-0.70.19-py312-none-any.whl", hash = "sha256:3a56c0e85dd5025161bac5ce138dcac1e49174c7d8e74596537e729fd5c53c28", size = 150281, upload-time = "2026-01-19T06:47:35.037Z" }, + { url = "https://files.pythonhosted.org/packages/7f/74/d2c27e03cb84251dfe7249b8e82923643c6d48fa4883b9476b025e7dc7eb/multiprocess-0.70.19-py313-none-any.whl", hash = "sha256:8d5eb4ec5017ba2fab4e34a747c6d2c2b6fecfe9e7236e77988db91580ada952", size = 156414, upload-time = "2026-01-19T06:47:35.915Z" }, + { url = "https://files.pythonhosted.org/packages/a0/61/af9115673a5870fd885247e2f1b68c4f1197737da315b520a91c757a861a/multiprocess-0.70.19-py314-none-any.whl", hash = "sha256:e8cc7fbdff15c0613f0a1f1f8744bef961b0a164c0ca29bdff53e9d2d93c5e5f", size = 160318, upload-time = "2026-01-19T06:47:37.497Z" }, + { url = "https://files.pythonhosted.org/packages/7e/82/69e539c4c2027f1e1697e09aaa2449243085a0edf81ae2c6341e84d769b6/multiprocess-0.70.19-py39-none-any.whl", hash = "sha256:0d4b4397ed669d371c81dcd1ef33fd384a44d6c3de1bd0ca7ac06d837720d3c5", size = 133477, upload-time = "2026-01-19T06:47:38.619Z" }, +] + +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + +[[package]] +name = "nltk" +version = "3.10.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "defusedxml" }, + { name = "joblib" }, + { name = "regex" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4a/65/20fa203b28b258fa1222305593ca281e4ad33729c389676bc0d29a8856fd/nltk-3.10.1.tar.gz", hash = "sha256:86a1b41d9ca0d35a2cb72fa60af4c9aaba9fe405b717161fd94cecd69f467007", size = 3098602, upload-time = "2026-08-01T06:25:20.748Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/47/44ffb39cb0edf6b7164fdd87441044d0a1924f0a2d8470e1ad0f533711e0/nltk-3.10.1-py3-none-any.whl", hash = "sha256:55b8780b6b97732c1c3806d4ae02d46113204b11bfdc19dddb95729f627f8853", size = 1725226, upload-time = "2026-08-01T06:25:08.199Z" }, +] + +[[package]] +name = "numpy" +version = "2.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/fd/89965aa4ac08c74998539fcbf24fa3540f3e15237fbeb6bcf9c908f4aade/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3", size = 20755553, upload-time = "2026-07-04T17:08:00.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/7b/14687aa674250e5e546f616f486b0d56d3631cd5b2415739141ce40bdcea/numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277", size = 16801574, upload-time = "2026-07-04T17:06:12.423Z" }, + { url = "https://files.pythonhosted.org/packages/e1/19/cc5bb2a3f2913d27d6dbb2c78d25921fabaedc6741d4a5a615a11f3c5bf3/numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1", size = 11772250, upload-time = "2026-07-04T17:06:15.726Z" }, + { url = "https://files.pythonhosted.org/packages/42/77/fdf34a71dd30f54979b18603bee915e0aaf825b07afe79acd60b04b691e2/numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0", size = 5331516, upload-time = "2026-07-04T17:06:17.913Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e2/eb7efa015b4cce41e2517bf182a7fce0d7d5b9d9ed76a29bfa0f4fe4505c/numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e", size = 6664863, upload-time = "2026-07-04T17:06:20.02Z" }, + { url = "https://files.pythonhosted.org/packages/a9/4b/a2b32dd94ee9ffbeecb28152240042a3949db33b1c834d44090b80e1b3b8/numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75", size = 15167977, upload-time = "2026-07-04T17:06:21.621Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a9/6e73d68500f80773f65f0654ea932019d6694329a0eb0ed0533de38df376/numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca", size = 16672469, upload-time = "2026-07-04T17:06:24.064Z" }, + { url = "https://files.pythonhosted.org/packages/24/7d/ad3e59015135f5261c95fd4cafeff159c955febd83a99a1d9250c4233815/numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3", size = 16527531, upload-time = "2026-07-04T17:06:26.69Z" }, + { url = "https://files.pythonhosted.org/packages/83/d0/a39b2fbcde9cb17a1dac678f254b33a6336298af9df338824c685425d5e8/numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9", size = 18431940, upload-time = "2026-07-04T17:06:29.521Z" }, + { url = "https://files.pythonhosted.org/packages/04/12/cff070947791c1ed425ff76413189adbdc2fbe215eba7ce7fa454a03c7f8/numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2", size = 6066764, upload-time = "2026-07-04T17:06:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/65/66/53f31807a48a750f9d748da273bc3fcedd12b27ff1f3e373bfec55ef2dc0/numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2", size = 12430966, upload-time = "2026-07-04T17:06:34.926Z" }, + { url = "https://files.pythonhosted.org/packages/2b/2a/d1a88066b1c14186f5d3c0d18c94f17b064511982bab0578d49ee9d43c29/numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b", size = 10350488, upload-time = "2026-07-04T17:06:37.785Z" }, + { url = "https://files.pythonhosted.org/packages/eb/07/ec2a3f0c91761581d4b7104a740791800025983f9a4dc4e73f91a99aeac4/numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1", size = 16796419, upload-time = "2026-07-04T17:06:40.37Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ab/ddb499fc4f8780354395face5b65c7fd107bcd6e1d667a5f07d046956f6f/numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6", size = 11765832, upload-time = "2026-07-04T17:06:42.768Z" }, + { url = "https://files.pythonhosted.org/packages/88/b3/3c28c558a09fc72100c646dac6d2fce8e834c471b0edca01a29996706117/numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d", size = 5325143, upload-time = "2026-07-04T17:06:45.466Z" }, + { url = "https://files.pythonhosted.org/packages/5e/0e/ce19b985bb15c596f4f05954e76cccc77c845083b3b8f938a6c68e523128/numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1", size = 6659749, upload-time = "2026-07-04T17:06:47.288Z" }, + { url = "https://files.pythonhosted.org/packages/2e/20/1ee6614d64332a1bba6411f38e68cb79eec1b2459e20a623777c5c5492a2/numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd", size = 15164716, upload-time = "2026-07-04T17:06:49.494Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a7/2bcd3fdbb87804755c35b729bf8709d62025c5f4cfd7d5b2415997097515/numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a", size = 16661440, upload-time = "2026-07-04T17:06:52.061Z" }, + { url = "https://files.pythonhosted.org/packages/fc/d7/a41e3310c886fe457d36e670bbf24fae411aca8a7b6ad92a32afd924077c/numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7", size = 16526305, upload-time = "2026-07-04T17:06:54.605Z" }, + { url = "https://files.pythonhosted.org/packages/53/75/4333a9a707c1edd3a4e1a0c58eca52c0f31e55089fa80db02b5565b24df7/numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6", size = 18423008, upload-time = "2026-07-04T17:06:57.54Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/e314a32b1c11a2ffe818ddad3a57b50b4b6e1b6c487192eb50cdef0415d0/numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9", size = 6063885, upload-time = "2026-07-04T17:07:00.14Z" }, + { url = "https://files.pythonhosted.org/packages/10/70/800b3fca480af32df9e8ea9f3d4a0c8feb4b32d7f195d174eabbda4829ad/numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74", size = 12425674, upload-time = "2026-07-04T17:07:02.387Z" }, + { url = "https://files.pythonhosted.org/packages/8b/0b/196350c122f50f6ca56846f2d71efd5e0d24b7b2e07355e019b2e2c7a11e/numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107", size = 10350256, upload-time = "2026-07-04T17:07:04.878Z" }, + { url = "https://files.pythonhosted.org/packages/db/f4/731b6085a83faf6ca843394cbd5e217280c214399f7e8b21b9f552af0ae2/numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8", size = 16795063, upload-time = "2026-07-04T17:07:07.374Z" }, + { url = "https://files.pythonhosted.org/packages/bf/64/0e215f2048dd11a55bb989ed41b3585ef57452404e638d703a211a3e4157/numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75", size = 11776652, upload-time = "2026-07-04T17:07:09.907Z" }, + { url = "https://files.pythonhosted.org/packages/b5/59/2b844c7a6e9deff69b404a66221e1542937734f65d5e6e39411876053862/numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2", size = 5335944, upload-time = "2026-07-04T17:07:12.227Z" }, + { url = "https://files.pythonhosted.org/packages/86/51/9bf7cb2cabcebc9e017e4ec7e6322b378317a542c08b4cb68479c1efc716/numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b", size = 6656266, upload-time = "2026-07-04T17:07:14.368Z" }, + { url = "https://files.pythonhosted.org/packages/83/3e/fb7615b211b82a32f44d5180a6d421b61f84d4fadd578b48ba4ac34e189f/numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95", size = 15179720, upload-time = "2026-07-04T17:07:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/41/5f/0f992cb24560673496c5d68de61913b57166ce530ffda07c1f280e0cc464/numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21", size = 16664835, upload-time = "2026-07-04T17:07:19.021Z" }, + { url = "https://files.pythonhosted.org/packages/a2/2f/97d6475ee91afe2587797d09446f9d3e475ad4cb681662d824809327b75a/numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373", size = 16539135, upload-time = "2026-07-04T17:07:22.015Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5b/4db81e4ba0be7e2776b1de68c82aa862c7f8ec27e1b4927d4ae075e20678/numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438", size = 18426684, upload-time = "2026-07-04T17:07:24.941Z" }, + { url = "https://files.pythonhosted.org/packages/1f/64/c0ba2d90724d450279a7df8f32057241070250a26a7e2b5337d77347f481/numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace", size = 6116103, upload-time = "2026-07-04T17:07:27.622Z" }, + { url = "https://files.pythonhosted.org/packages/c1/1a/837f9ed7405adcd7a40538792eb169eddd8fa5630c16a1ef49dae71a30f4/numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a", size = 12562177, upload-time = "2026-07-04T17:07:29.887Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/49707938b6dd0a78a9178dd93227dc89e4c11af47f5c798d70366e8d0483/numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0", size = 10627739, upload-time = "2026-07-04T17:07:32.568Z" }, + { url = "https://files.pythonhosted.org/packages/a6/c7/bb4b882cfe7f299cbc8b66e42e7dd78cf9d14e40f9469fc5e3db7e15b3bd/numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22", size = 11894709, upload-time = "2026-07-04T17:07:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/40/3f/5af7f4a7f6224aef48017aa82bb6174c7a659d724be0c75017b7e64a55b4/numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7", size = 5453810, upload-time = "2026-07-04T17:07:37.495Z" }, + { url = "https://files.pythonhosted.org/packages/20/c9/3474309bc94d634d3f9c3eddf03250ecb8c22cd948ef16fef69a77cc5d7b/numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d", size = 6761189, upload-time = "2026-07-04T17:07:39.563Z" }, + { url = "https://files.pythonhosted.org/packages/90/8a/558ae39fdd55d7e7f7fef9a84a6e964ac6b23edbd2a07e52bb084500507d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09", size = 15225039, upload-time = "2026-07-04T17:07:41.682Z" }, + { url = "https://files.pythonhosted.org/packages/63/27/ca7392b2d030277bdf0273e7d23255b3ee57d57a7c170a6f4fb3981e1e5d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4", size = 16701306, upload-time = "2026-07-04T17:07:44.611Z" }, + { url = "https://files.pythonhosted.org/packages/02/42/03d53ae7996c44d4374a8262e9dc41671fd56cbb98f7d47ef85cf5da4c6b/numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1", size = 16589955, upload-time = "2026-07-04T17:07:47.694Z" }, + { url = "https://files.pythonhosted.org/packages/7b/15/6c1784ae469640e65db111e9a34b3d0f14d91e8a38b9ce34810ced370dbb/numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077", size = 18464252, upload-time = "2026-07-04T17:07:50.684Z" }, + { url = "https://files.pythonhosted.org/packages/94/a8/f98e50356cf167df656c526c2dfeec2d7dde182f2a3da4b458a5938e2776/numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf", size = 6263298, upload-time = "2026-07-04T17:07:53.445Z" }, + { url = "https://files.pythonhosted.org/packages/72/ac/96ae880cdecad0b3275d9359fcec72667b49a4863c9f12942e43679dda02/numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af", size = 12748623, upload-time = "2026-07-04T17:07:55.384Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5a/4d2b1601df3602dba7a14f3348ba9bfe94a18adb428e693df6154c293831/numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb", size = 10697674, upload-time = "2026-07-04T17:07:58.506Z" }, +] + +[[package]] +name = "olmocr" +version = "0.4.27" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "bleach" }, + { name = "boto3" }, + { name = "cached-path" }, + { name = "cryptography" }, + { name = "filelock" }, + { name = "ftfy" }, + { name = "httpx" }, + { name = "lingua-language-detector" }, + { name = "markdown2" }, + { name = "markdownify" }, + { name = "orjson" }, + { name = "pillow" }, + { name = "pypdf" }, + { name = "pypdfium2" }, + { name = "requests" }, + { name = "smart-open" }, + { name = "zstandard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d0/2c/85d7a6c032cbad5a5d25178618a2704225fcfd692340c9d88ab1d14705fc/olmocr-0.4.27.tar.gz", hash = "sha256:7da74f37a3e987f966765503c59913c2268289c9ecb14b4c5f40c89a0e8e5393", size = 410438, upload-time = "2026-03-12T16:39:26.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/5c/41b86d46c037dc5bc1fceaa5843c8029ebc54c6713afe4a9e10c163c4cc1/olmocr-0.4.27-py3-none-any.whl", hash = "sha256:4c54b77d1e5dd487bcd60be26728dc07aa74f7fb3947bae4c697776bd500f480", size = 423408, upload-time = "2026-03-12T16:39:25.349Z" }, +] + +[[package]] +name = "openai" +version = "2.53.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/cf/36e3e7235fdf6d125c052acc0970924611b17a20a4fe580596faf4566a65/openai-2.53.0.tar.gz", hash = "sha256:baf5802ad08980e1d9d561e1b996e800c8bcd14af5847c6d0e7a5cc59e4d4116", size = 1099435, upload-time = "2026-08-03T21:42:01.664Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/0f/cc6afea3542a5142c5d8fc8211c5e059a8375105d004a41dfa2c7948dbb0/openai-2.53.0-py3-none-any.whl", hash = "sha256:c694ffc747a3c4d1663ef2b07b811315a476164ee5efa3a993967349ebca7618", size = 1659829, upload-time = "2026-08-03T21:41:59.581Z" }, +] + +[[package]] +name = "opencv-python" +version = "5.0.0.93" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/4c/a438d23e09ce2033c09f7b784ad2fbdb0adf529e434101ed28f142226f98/opencv_python-5.0.0.93.tar.gz", hash = "sha256:66aac3e5b5faa48d4025816592f3af19e4bfc2c68dec067bae2dbb4ca10aa9e2", size = 81802749, upload-time = "2026-07-02T06:59:53.815Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/75/76f6ade78f6102c61034f828e2a22616708df2c9504bc8d6af9dd8f73dc5/opencv_python-5.0.0.93-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:198a75138241810206a17c829dbcc40a7cb1841cda538ca86cbbfc6c7d95f898", size = 48322443, upload-time = "2026-07-02T05:50:25.466Z" }, + { url = "https://files.pythonhosted.org/packages/15/8c/bc1bda6aae69a32e9d84fc34153ba104cd25226861eb4aea33b2cea4860d/opencv_python-5.0.0.93-cp37-abi3-macosx_14_0_x86_64.whl", hash = "sha256:6bbc32f59e1b1a7db7b39c81f63d00625f041d333037fd8702f6da52cc39108b", size = 34782755, upload-time = "2026-07-02T05:51:30.556Z" }, + { url = "https://files.pythonhosted.org/packages/f4/8a/b04776ec45d2dea08a1b176f1829201db3515d4ed16c35f8fcc9fa7beb16/opencv_python-5.0.0.93-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e2b4272e736836f66c2d176e43ab8101f3a00d45654916399f52e150c58981ac", size = 50614064, upload-time = "2026-07-02T06:53:22.604Z" }, + { url = "https://files.pythonhosted.org/packages/95/54/eb47866b94f2b5b42dde17644b78055ef1ee05aae59962c7290e55270803/opencv_python-5.0.0.93-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f8b6d0a212253dd26ad338c812f1f23ca118fdf05a9c8c6b9444f161aa8c5881", size = 71064711, upload-time = "2026-07-02T06:54:13.148Z" }, + { url = "https://files.pythonhosted.org/packages/93/da/962579f1e703cbf8c5422fd1f576467dcb3b5b0b0b81c1471c979764353a/opencv_python-5.0.0.93-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:08d5d91d967b58d6db86073b2ad3eaef88ca4ebdfd45c9059bf59f5ded0c7ad2", size = 49798576, upload-time = "2026-07-02T06:54:33.781Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4c/c73f828fdbcd37eaf21d08fa852544a3ca7c2dbb3ea76873d64f2ea413d1/opencv_python-5.0.0.93-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:c8de2dec111122a02e8beb28e16c31904992dfd6186560b142a92c71403c1039", size = 73783032, upload-time = "2026-07-02T06:55:03.415Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4b/edaf83b996ca5a1a3d8ccad485706b9c6d4742b13b9c4586bf1c1e7d9423/opencv_python-5.0.0.93-cp37-abi3-win32.whl", hash = "sha256:4b4b1a34c79bf8d3738e3cfe9a9e67b51a79663f6b692cbdad8c31f570da4157", size = 35564734, upload-time = "2026-07-02T05:49:57.704Z" }, + { url = "https://files.pythonhosted.org/packages/21/f0/9fa6e85cb10c8eb36a0222d27e50fe381b86ce49a55446bf39f491727564/opencv_python-5.0.0.93-cp37-abi3-win_amd64.whl", hash = "sha256:f90ba04b8f73bc5c3814037699739f0156f597338a98f05956c684e7c3ca10d2", size = 44000345, upload-time = "2026-07-02T05:49:54.971Z" }, +] + +[[package]] +name = "orjson" +version = "3.11.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/0c/964746fcafbd16f8ff53219ad9f6b412b34f345c75f384ad434ceaadb538/orjson-3.11.9.tar.gz", hash = "sha256:4fef17e1f8722c11587a6ef18e35902450221da0028e65dbaaa543619e68e48f", size = 5599163, upload-time = "2026-05-06T15:11:08.309Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/6d/11867a3ffa3a3608d84a4de51ef4dd0896d6b5cc9132fbe1daf593e677bc/orjson-3.11.9-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9ef6fe90aadef185c7b128859f40beb24720b4ecea95379fc9000931179c3a49", size = 228515, upload-time = "2026-05-06T15:09:57.265Z" }, + { url = "https://files.pythonhosted.org/packages/24/75/05912954c8b288f34fcf5cd4b9b071cb4f6e77b9961e175e56ebb258089f/orjson-3.11.9-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:e5c9b8f28e726e97d97696c826bc7bea5d71cecd63576dba92924a32c1961291", size = 128409, upload-time = "2026-05-06T15:09:59.063Z" }, + { url = "https://files.pythonhosted.org/packages/ab/86/1c3a47df3bc8191ea9ac51603bbb872a95167a364320c269f2557911f406/orjson-3.11.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26a473dbb4162108b27901492546f83c76fdcea3d0eadff00ae7a07e18dcce09", size = 132106, upload-time = "2026-05-06T15:10:00.798Z" }, + { url = "https://files.pythonhosted.org/packages/d7/cf/b33b5f3e695ae7d63feef9d915c37cc3b8f465493dcd4f8e0b4c697a2366/orjson-3.11.9-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:011382e2a60fda9d46f1cdee31068cfc52ffe952b587d683ec0463002802a0f4", size = 127864, upload-time = "2026-05-06T15:10:02.15Z" }, + { url = "https://files.pythonhosted.org/packages/31/6a/6cf69385a58208024fcb8c014e2141b8ce838aba6492b589f8acfff97fab/orjson-3.11.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2d3dc759490128c5c1711a53eeaa8ee1d437fd0038ffd2b6008abf46db3f882", size = 135213, upload-time = "2026-05-06T15:10:03.515Z" }, + { url = "https://files.pythonhosted.org/packages/e8/f8/0b1bd3e8f2efcdd376af5c8cfd79eaf13f018080c0089c80ebd724e3c7fb/orjson-3.11.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8ea516b3726d190e1b4297e6f4e7a8650347ae053868a18163b4dd3641d1fff", size = 145994, upload-time = "2026-05-06T15:10:05.083Z" }, + { url = "https://files.pythonhosted.org/packages/f3/59/dab79f61044c529d2c81aecdc589b1f833a1c8dec11ba3b1c2498a02ca7e/orjson-3.11.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380cdce7ba24989af81d0a7013d0aaec5d0e2a21734c0e2681b1bc4f141957fe", size = 132744, upload-time = "2026-05-06T15:10:06.853Z" }, + { url = "https://files.pythonhosted.org/packages/0e/a4/82b7a2fe5d8a67a59ed831b24d59a3d46ea7d207b66e1602d376541d94a6/orjson-3.11.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be4fa4f0af7fa18951f7ab3fc2148e223af211bf03f59e1c6034ec3f97f21d61", size = 134014, upload-time = "2026-05-06T15:10:08.213Z" }, + { url = "https://files.pythonhosted.org/packages/50/c7/375e83a76851b73b2e39f3bcf0e5a19e2b89bad13e5bca97d0b293d27f24/orjson-3.11.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a8f5f8bc7ce7d59f08d9f99fa510c06496164a24cb5f3d34537dbd9ca30132e2", size = 141509, upload-time = "2026-05-06T15:10:09.595Z" }, + { url = "https://files.pythonhosted.org/packages/7f/7c/49d5d82a3d3097f641f094f552131f1e2723b0b8cb0fa2874ab65ecfffa6/orjson-3.11.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4d7fde5501b944f83b3e665e1b31343ff6e154b15560a16b7130ea1e594a4206", size = 415127, upload-time = "2026-05-06T15:10:11.049Z" }, + { url = "https://files.pythonhosted.org/packages/3a/dc/7446c538590d55f455647e5f3c61fc33f7108714e7afcffa6a2a033f8350/orjson-3.11.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cde1a448023ba7d5bb4c01c5afb48894380b5e4956e0627266526587ef4e535f", size = 148025, upload-time = "2026-05-06T15:10:12.842Z" }, + { url = "https://files.pythonhosted.org/packages/df/e5/4d2d8af06f788329b4f78f8cc3679bb395392fcaa1e4d8d3c33e85308fa4/orjson-3.11.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e63adb0e1f1ed5d9e168f50a91ceb93ae6420731d222dc7da5c69409aa47aa", size = 136943, upload-time = "2026-05-06T15:10:14.405Z" }, + { url = "https://files.pythonhosted.org/packages/06/69/850264ccf6d80f6b174620d30a87f65c9b1490aba33fe6b62798e618cad3/orjson-3.11.9-cp312-cp312-win32.whl", hash = "sha256:2d057a602cdd19a0ad680417527c45b6961a095081c0f46fe0e03e304aac6470", size = 131606, upload-time = "2026-05-06T15:10:15.791Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d5/973a43fc9c55e20f2051e9830997649f669be0cb3ca52192087c0143f118/orjson-3.11.9-cp312-cp312-win_amd64.whl", hash = "sha256:59e403b1cc5a676da8eaf31f6254801b7341b3e29efa85f92b48d272637e77be", size = 127101, upload-time = "2026-05-06T15:10:17.129Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ae/495470f0e4a18f73fa10b7f6b84b464ec4cc5291c4e0c7c2a6c400bef006/orjson-3.11.9-cp312-cp312-win_arm64.whl", hash = "sha256:9af678d6488357948f1f84c6cd1c1d397c014e1ae2f98ae082a44eb48f602624", size = 126736, upload-time = "2026-05-06T15:10:18.645Z" }, + { url = "https://files.pythonhosted.org/packages/32/33/93fcc25907235c344ae73122f8a4e01d2d393ef062b4af7d2e2487a32c37/orjson-3.11.9-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4bab1b2d6141fe7b32ae71dac905666ece4f94936efbfb13d55bb7739a3a6021", size = 228458, upload-time = "2026-05-06T15:10:20.079Z" }, + { url = "https://files.pythonhosted.org/packages/8f/27/b1e6dadb3c080313c03fdd8067b85e6a0460c7d8d6a1c3984ef77b904e4d/orjson-3.11.9-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:844417969855fc7a41be124aafe83dc424592a7f77cd4501900c67307122b92c", size = 128368, upload-time = "2026-05-06T15:10:21.549Z" }, + { url = "https://files.pythonhosted.org/packages/21/0f/c9ede0bf052f6b4051e64a7d4fa91b725cccf8321a6a786e86eb03519f00/orjson-3.11.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffe02797b5e9f3a9d8292ddcd289b474ad13e81ad83cd1891a240811f1d2cb81", size = 132070, upload-time = "2026-05-06T15:10:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/fd/26/d398e28048dc18205bbe812f2c88cb9b40313db2470778e25964796458fe/orjson-3.11.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e4eed3b200023042814d2fc8a5d2e880f13b52e1ed2485e83da4f3962f7dc1a", size = 127892, upload-time = "2026-05-06T15:10:24.714Z" }, + { url = "https://files.pythonhosted.org/packages/66/60/52b0054c4c700d5aa7fc5b7ca96917400d8f061307778578e67a10e25852/orjson-3.11.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aff7da9952a5ad1cef8e68017724d96c7b9a66e99e91d6252e1b133d67a7b10", size = 135217, upload-time = "2026-05-06T15:10:26.084Z" }, + { url = "https://files.pythonhosted.org/packages/d5/97/1e3dc2b2a28b7b2528f403d2fc1d79ec5f39af3bc143ab65d3ec26426385/orjson-3.11.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d4e98d6f3b8afed8bc8cd9718ec0cdf46661826beefb53fe8eafb37f2bf0362", size = 145980, upload-time = "2026-05-06T15:10:28.062Z" }, + { url = "https://files.pythonhosted.org/packages/fc/39/31fbfe7850f2de32dee7e7e5c09f26d403ab01e440ac96001c6b01ad3c99/orjson-3.11.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a81d52442a7c99b3662333235b3adf96a1715864658b35bb797212be7bddb97", size = 132738, upload-time = "2026-05-06T15:10:29.727Z" }, + { url = "https://files.pythonhosted.org/packages/a1/08/dca0082dd2a194acb93e5457e73455388e2e2ca464a2672449a9ddbb679d/orjson-3.11.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e39364e726a8fff737309aff059ff67d8a8c8d5b677be7bb49a8b3e84b7e218", size = 134033, upload-time = "2026-05-06T15:10:31.152Z" }, + { url = "https://files.pythonhosted.org/packages/11/d4/5bdb0626801230139987385554c5d4c42255218ac906525bf4347f22cd95/orjson-3.11.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4fd66214623f1b17501df9f0543bef0b833979ab5b6ded1e1d123222866aa8c9", size = 141492, upload-time = "2026-05-06T15:10:32.641Z" }, + { url = "https://files.pythonhosted.org/packages/fa/88/a21fb53b3ede6703aede6dce4710ed4111e5b201cfa6bbff5e544f9d47d7/orjson-3.11.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8ecc30f10465fa1e0ce13fd01d9e22c316e5053a719a8d915d4545a09a5ff677", size = 415087, upload-time = "2026-05-06T15:10:34.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/57/1b30daf70f0d8180e9a73cefbfbdd99e4bf19eb020466502b01fba7e0e50/orjson-3.11.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:97db4c94a7db398a5bd636273324f0b3fd58b350bbbac8bb380ceb825a9b40f4", size = 148031, upload-time = "2026-05-06T15:10:36.358Z" }, + { url = "https://files.pythonhosted.org/packages/04/83/45fbb6d962e260807f99441db9613cee868ceda4baceda59b3720a563f97/orjson-3.11.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f78cf8fec5bd627f4082b8dfeac7871b43d7f3274904492a43dab39f18a19a0", size = 136915, upload-time = "2026-05-06T15:10:38.013Z" }, + { url = "https://files.pythonhosted.org/packages/5f/cc/2d10025f9056d376e4127ec05a5808b218d46f035fdc08178a5411b34250/orjson-3.11.9-cp313-cp313-win32.whl", hash = "sha256:d4087e5c0209a0a8efe4de3303c234b9c44d1174161dcd851e8eea07c7560b32", size = 131613, upload-time = "2026-05-06T15:10:39.569Z" }, + { url = "https://files.pythonhosted.org/packages/67/bd/2775ff28bfe883b9aa1ff348300542eb2ef1ee18d8ae0e3a49846817a865/orjson-3.11.9-cp313-cp313-win_amd64.whl", hash = "sha256:051b102c93b4f634e89f3866b07b9a9a98915ada541f4ec30f177067b2694979", size = 127086, upload-time = "2026-05-06T15:10:41.262Z" }, + { url = "https://files.pythonhosted.org/packages/91/2b/d26799e580939e32a7da9a39531bc9e58e15ca32ffaa6a8cb3e9bb0d22cd/orjson-3.11.9-cp313-cp313-win_arm64.whl", hash = "sha256:cce9127885941bd28f080cecf1f1d288336b7e0d812c345b08be88b572796254", size = 126696, upload-time = "2026-05-06T15:10:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/8e/eb/5da01e356015aee6ecfa1187ced87aef51364e306f5e695dd52719bf0e78/orjson-3.11.9-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b6ef1979adc4bc243523f1a2ba91418030a8e29b0a99cbe7e0e2d6807d4dce6e", size = 228465, upload-time = "2026-05-06T15:10:44.097Z" }, + { url = "https://files.pythonhosted.org/packages/64/62/3e0e0c14c957133bcd855395c62b55ed4e3b0af23ffea11b032cb1dcbdb1/orjson-3.11.9-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:f36b7f32c7c0db4a719f1fc5824db4a9c6f8bd1a354debb91faf26ebf3a4c71e", size = 128364, upload-time = "2026-05-06T15:10:45.839Z" }, + { url = "https://files.pythonhosted.org/packages/5a/5a/07d8aa117211a8ed7630bda80c8c0b14d04e0f8dcf99bcf49656e4a710eb/orjson-3.11.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08f4d8ebb44925c794e535b2bebc507cebf32209df81de22ae285fb0d8d66de0", size = 132063, upload-time = "2026-05-06T15:10:47.267Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ec/4acaf21483e18aa945be74a474c74b434f284b549f275a0a39b9f98956e9/orjson-3.11.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6cc7923789694fd58f001cbcac7e47abc13af4d560ebbfcf3b41a8b1a0748124", size = 122356, upload-time = "2026-05-06T15:10:48.765Z" }, + { url = "https://files.pythonhosted.org/packages/13/d8/5f0555e7638801323b7a75850f92e7dfa891bc84fe27a1ba4449170d1200/orjson-3.11.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea5c46eb2d3af39e806b986f4b09d5c2706a1f5afde3cbf7544ce6616127173c", size = 129592, upload-time = "2026-05-06T15:10:50.13Z" }, + { url = "https://files.pythonhosted.org/packages/b6/30/ed9860412a3603ceb3c5955bfd72d28b9d0e7ba6ed81add14f83d7114236/orjson-3.11.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f5d89a2ed90731df3be64bab0aa44f78bff39fdc9d71c291f4a8023aa46425b7", size = 140491, upload-time = "2026-05-06T15:10:51.582Z" }, + { url = "https://files.pythonhosted.org/packages/d0/17/adc514dea7ac7c505527febf884934b815d34f0c7b8693c1a8b39c5c4a57/orjson-3.11.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25e4aed0312d292c09f61af25bba34e0b2c88546041472b09088c39a4d828af1", size = 127309, upload-time = "2026-05-06T15:10:53.329Z" }, + { url = "https://files.pythonhosted.org/packages/76/3e/c0b690253f0b82d86e99949af13533363acfb5432ecb5d53dd5b3bce9c34/orjson-3.11.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaea64f3f467d22e70eeed68bdccb3bc4f83f650446c4a03c59f2cba28a108db", size = 134030, upload-time = "2026-05-06T15:10:54.988Z" }, + { url = "https://files.pythonhosted.org/packages/c1/7a/bc82a0bb25e9faaf92dc4d9ef002732efc09737706af83e346788641d4a7/orjson-3.11.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a028425d1b440c5d92a6be1e1a020739dfe67ea87d96c6dbe828c1b30041728b", size = 141482, upload-time = "2026-05-06T15:10:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/01/55/e69188b939f77d5d32a9833745ace31ea5ccae3ab613a1ec185d3cd2c4fb/orjson-3.11.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5b192c6cf397e4455b11523c5cf2b18ed084c1bbd61b6c0926344d2129481972", size = 415178, upload-time = "2026-05-06T15:10:58.446Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1a/b8a5a7ac527e80b9cb11d51e3f6689b709279183264b9ec5c7bc680bb8b5/orjson-3.11.9-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea407d4ccf5891d667d045fecae97a7a1e5e87b3b97f97ae1803c2e741130be0", size = 148089, upload-time = "2026-05-06T15:11:00.441Z" }, + { url = "https://files.pythonhosted.org/packages/97/4e/00503f64204bf859b37213a63927028f30fb6268cd8677fb0a5ad48155e1/orjson-3.11.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f63aaf97afd9f6dec5b1a68e1b8da12bfccb4cb9a9a65c3e0b6c847849e7586", size = 136921, upload-time = "2026-05-06T15:11:02.176Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ba/a23b82a0a8d0ed7bed4e5f5035aae751cad4ff6a1e8d2ecd14d8860f5929/orjson-3.11.9-cp314-cp314-win32.whl", hash = "sha256:e30ab17845bb9fa54ccf67fa4f9f5282652d54faa6d17452f47d0f369d038673", size = 131638, upload-time = "2026-05-06T15:11:03.696Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c3/0c6798456bade745c75c452342dabacce5798196483e77e643be1f53877d/orjson-3.11.9-cp314-cp314-win_amd64.whl", hash = "sha256:32ef5f4283a3be81913947d19608eacb7c6608026851123790cd9cc8982af34b", size = 127078, upload-time = "2026-05-06T15:11:05.123Z" }, + { url = "https://files.pythonhosted.org/packages/16/21/5a3f1e8913103b703a436a5664238e5b965ec392b555fe68943ea3691e6b/orjson-3.11.9-cp314-cp314-win_arm64.whl", hash = "sha256:eebdbdeef0094e4f5aefa20dcd4eb2368ab5e7a3b4edea27f1e7b2892e009cf9", size = 126687, upload-time = "2026-05-06T15:11:06.602Z" }, +] + +[[package]] +name = "ormsgpack" +version = "1.12.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/12/0c/f1761e21486942ab9bb6feaebc610fa074f7c5e496e6962dea5873348077/ormsgpack-1.12.2.tar.gz", hash = "sha256:944a2233640273bee67521795a73cf1e959538e0dfb7ac635505010455e53b33", size = 39031, upload-time = "2026-01-18T20:55:28.023Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/36/16c4b1921c308a92cef3bf6663226ae283395aa0ff6e154f925c32e91ff5/ormsgpack-1.12.2-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:7a29d09b64b9694b588ff2f80e9826bdceb3a2b91523c5beae1fab27d5c940e7", size = 378618, upload-time = "2026-01-18T20:55:50.835Z" }, + { url = "https://files.pythonhosted.org/packages/c0/68/468de634079615abf66ed13bb5c34ff71da237213f29294363beeeca5306/ormsgpack-1.12.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b39e629fd2e1c5b2f46f99778450b59454d1f901bc507963168985e79f09c5d", size = 203186, upload-time = "2026-01-18T20:56:11.163Z" }, + { url = "https://files.pythonhosted.org/packages/73/a9/d756e01961442688b7939bacd87ce13bfad7d26ce24f910f6028178b2cc8/ormsgpack-1.12.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:958dcb270d30a7cb633a45ee62b9444433fa571a752d2ca484efdac07480876e", size = 210738, upload-time = "2026-01-18T20:56:09.181Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ba/795b1036888542c9113269a3f5690ab53dd2258c6fb17676ac4bd44fcf94/ormsgpack-1.12.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58d379d72b6c5e964851c77cfedfb386e474adee4fd39791c2c5d9efb53505cc", size = 212569, upload-time = "2026-01-18T20:56:06.135Z" }, + { url = "https://files.pythonhosted.org/packages/6c/aa/bff73c57497b9e0cba8837c7e4bcab584b1a6dbc91a5dd5526784a5030c8/ormsgpack-1.12.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8463a3fc5f09832e67bdb0e2fda6d518dc4281b133166146a67f54c08496442e", size = 387166, upload-time = "2026-01-18T20:55:36.738Z" }, + { url = "https://files.pythonhosted.org/packages/d3/cf/f8283cba44bcb7b14f97b6274d449db276b3a86589bdb363169b51bc12de/ormsgpack-1.12.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:eddffb77eff0bad4e67547d67a130604e7e2dfbb7b0cde0796045be4090f35c6", size = 482498, upload-time = "2026-01-18T20:55:29.626Z" }, + { url = "https://files.pythonhosted.org/packages/05/be/71e37b852d723dfcbe952ad04178c030df60d6b78eba26bfd14c9a40575e/ormsgpack-1.12.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fcd55e5f6ba0dbce624942adf9f152062135f991a0126064889f68eb850de0dd", size = 425518, upload-time = "2026-01-18T20:55:49.556Z" }, + { url = "https://files.pythonhosted.org/packages/7a/0c/9803aa883d18c7ef197213cd2cbf73ba76472a11fe100fb7dab2884edf48/ormsgpack-1.12.2-cp312-cp312-win_amd64.whl", hash = "sha256:d024b40828f1dde5654faebd0d824f9cc29ad46891f626272dd5bfd7af2333a4", size = 117462, upload-time = "2026-01-18T20:55:47.726Z" }, + { url = "https://files.pythonhosted.org/packages/c8/9e/029e898298b2cc662f10d7a15652a53e3b525b1e7f07e21fef8536a09bb8/ormsgpack-1.12.2-cp312-cp312-win_arm64.whl", hash = "sha256:da538c542bac7d1c8f3f2a937863dba36f013108ce63e55745941dda4b75dbb6", size = 111559, upload-time = "2026-01-18T20:55:54.273Z" }, + { url = "https://files.pythonhosted.org/packages/eb/29/bb0eba3288c0449efbb013e9c6f58aea79cf5cb9ee1921f8865f04c1a9d7/ormsgpack-1.12.2-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:5ea60cb5f210b1cfbad8c002948d73447508e629ec375acb82910e3efa8ff355", size = 378661, upload-time = "2026-01-18T20:55:57.765Z" }, + { url = "https://files.pythonhosted.org/packages/6e/31/5efa31346affdac489acade2926989e019e8ca98129658a183e3add7af5e/ormsgpack-1.12.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3601f19afdbea273ed70b06495e5794606a8b690a568d6c996a90d7255e51c1", size = 203194, upload-time = "2026-01-18T20:56:08.252Z" }, + { url = "https://files.pythonhosted.org/packages/eb/56/d0087278beef833187e0167f8527235ebe6f6ffc2a143e9de12a98b1ce87/ormsgpack-1.12.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29a9f17a3dac6054c0dce7925e0f4995c727f7c41859adf9b5572180f640d172", size = 210778, upload-time = "2026-01-18T20:55:17.694Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a2/072343e1413d9443e5a252a8eb591c2d5b1bffbe5e7bfc78c069361b92eb/ormsgpack-1.12.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39c1bd2092880e413902910388be8715f70b9f15f20779d44e673033a6146f2d", size = 212592, upload-time = "2026-01-18T20:55:32.747Z" }, + { url = "https://files.pythonhosted.org/packages/a2/8b/a0da3b98a91d41187a63b02dda14267eefc2a74fcb43cc2701066cf1510e/ormsgpack-1.12.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:50b7249244382209877deedeee838aef1542f3d0fc28b8fe71ca9d7e1896a0d7", size = 387164, upload-time = "2026-01-18T20:55:40.853Z" }, + { url = "https://files.pythonhosted.org/packages/19/bb/6d226bc4cf9fc20d8eb1d976d027a3f7c3491e8f08289a2e76abe96a65f3/ormsgpack-1.12.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:5af04800d844451cf102a59c74a841324868d3f1625c296a06cc655c542a6685", size = 482516, upload-time = "2026-01-18T20:55:42.033Z" }, + { url = "https://files.pythonhosted.org/packages/fb/f1/bb2c7223398543dedb3dbf8bb93aaa737b387de61c5feaad6f908841b782/ormsgpack-1.12.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cec70477d4371cd524534cd16472d8b9cc187e0e3043a8790545a9a9b296c258", size = 425539, upload-time = "2026-01-18T20:55:24.727Z" }, + { url = "https://files.pythonhosted.org/packages/7b/e8/0fb45f57a2ada1fed374f7494c8cd55e2f88ccd0ab0a669aa3468716bf5f/ormsgpack-1.12.2-cp313-cp313-win_amd64.whl", hash = "sha256:21f4276caca5c03a818041d637e4019bc84f9d6ca8baa5ea03e5cc8bf56140e9", size = 117459, upload-time = "2026-01-18T20:55:56.876Z" }, + { url = "https://files.pythonhosted.org/packages/7a/d4/0cfeea1e960d550a131001a7f38a5132c7ae3ebde4c82af1f364ccc5d904/ormsgpack-1.12.2-cp313-cp313-win_arm64.whl", hash = "sha256:baca4b6773d20a82e36d6fd25f341064244f9f86a13dead95dd7d7f996f51709", size = 111577, upload-time = "2026-01-18T20:55:43.605Z" }, + { url = "https://files.pythonhosted.org/packages/94/16/24d18851334be09c25e87f74307c84950f18c324a4d3c0b41dabdbf19c29/ormsgpack-1.12.2-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:bc68dd5915f4acf66ff2010ee47c8906dc1cf07399b16f4089f8c71733f6e36c", size = 378717, upload-time = "2026-01-18T20:55:26.164Z" }, + { url = "https://files.pythonhosted.org/packages/b5/a2/88b9b56f83adae8032ac6a6fa7f080c65b3baf9b6b64fd3d37bd202991d4/ormsgpack-1.12.2-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46d084427b4132553940070ad95107266656cb646ea9da4975f85cb1a6676553", size = 203183, upload-time = "2026-01-18T20:55:18.815Z" }, + { url = "https://files.pythonhosted.org/packages/a9/80/43e4555963bf602e5bdc79cbc8debd8b6d5456c00d2504df9775e74b450b/ormsgpack-1.12.2-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c010da16235806cf1d7bc4c96bf286bfa91c686853395a299b3ddb49499a3e13", size = 210814, upload-time = "2026-01-18T20:55:33.973Z" }, + { url = "https://files.pythonhosted.org/packages/78/e1/7cfbf28de8bca6efe7e525b329c31277d1b64ce08dcba723971c241a9d60/ormsgpack-1.12.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18867233df592c997154ff942a6503df274b5ac1765215bceba7a231bea2745d", size = 212634, upload-time = "2026-01-18T20:55:28.634Z" }, + { url = "https://files.pythonhosted.org/packages/95/f8/30ae5716e88d792a4e879debee195653c26ddd3964c968594ddef0a3cc7e/ormsgpack-1.12.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b009049086ddc6b8f80c76b3955df1aa22a5fbd7673c525cd63bf91f23122ede", size = 387139, upload-time = "2026-01-18T20:56:02.013Z" }, + { url = "https://files.pythonhosted.org/packages/dc/81/aee5b18a3e3a0e52f718b37ab4b8af6fae0d9d6a65103036a90c2a8ffb5d/ormsgpack-1.12.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1dcc17d92b6390d4f18f937cf0b99054824a7815818012ddca925d6e01c2e49e", size = 482578, upload-time = "2026-01-18T20:55:35.117Z" }, + { url = "https://files.pythonhosted.org/packages/bd/17/71c9ba472d5d45f7546317f467a5fc941929cd68fb32796ca3d13dcbaec2/ormsgpack-1.12.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f04b5e896d510b07c0ad733d7fce2d44b260c5e6c402d272128f8941984e4285", size = 425539, upload-time = "2026-01-18T20:56:04.009Z" }, + { url = "https://files.pythonhosted.org/packages/2e/a6/ac99cd7fe77e822fed5250ff4b86fa66dd4238937dd178d2299f10b69816/ormsgpack-1.12.2-cp314-cp314-win_amd64.whl", hash = "sha256:ae3aba7eed4ca7cb79fd3436eddd29140f17ea254b91604aa1eb19bfcedb990f", size = 117493, upload-time = "2026-01-18T20:56:07.343Z" }, + { url = "https://files.pythonhosted.org/packages/3a/67/339872846a1ae4592535385a1c1f93614138566d7af094200c9c3b45d1e5/ormsgpack-1.12.2-cp314-cp314-win_arm64.whl", hash = "sha256:118576ea6006893aea811b17429bfc561b4778fad393f5f538c84af70b01260c", size = 111579, upload-time = "2026-01-18T20:55:21.161Z" }, + { url = "https://files.pythonhosted.org/packages/49/c2/6feb972dc87285ad381749d3882d8aecbde9f6ecf908dd717d33d66df095/ormsgpack-1.12.2-cp314-cp314t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:7121b3d355d3858781dc40dafe25a32ff8a8242b9d80c692fd548a4b1f7fd3c8", size = 378721, upload-time = "2026-01-18T20:55:52.12Z" }, + { url = "https://files.pythonhosted.org/packages/a3/9a/900a6b9b413e0f8a471cf07830f9cf65939af039a362204b36bd5b581d8b/ormsgpack-1.12.2-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ee766d2e78251b7a63daf1cddfac36a73562d3ddef68cacfb41b2af64698033", size = 203170, upload-time = "2026-01-18T20:55:44.469Z" }, + { url = "https://files.pythonhosted.org/packages/87/4c/27a95466354606b256f24fad464d7c97ab62bce6cc529dd4673e1179b8fb/ormsgpack-1.12.2-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:292410a7d23de9b40444636b9b8f1e4e4b814af7f1ef476e44887e52a123f09d", size = 212816, upload-time = "2026-01-18T20:55:23.501Z" }, + { url = "https://files.pythonhosted.org/packages/73/cd/29cee6007bddf7a834e6cd6f536754c0535fcb939d384f0f37a38b1cddb8/ormsgpack-1.12.2-cp314-cp314t-win_amd64.whl", hash = "sha256:837dd316584485b72ef451d08dd3e96c4a11d12e4963aedb40e08f89685d8ec2", size = 117232, upload-time = "2026-01-18T20:55:45.448Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pandas" +version = "3.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/4f/5f3422a2afec5ffc46308b79e53291365a93748b498ac2e58bead0197916/pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712", size = 4658219, upload-time = "2026-07-22T22:19:28.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/54/1dc810ea558d1320b597aa140a514f2fdf1d2ea09c38cf556f13ea712ec9/pandas-3.0.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa290c16964d4963fbfbc358928239cf3bd755b20e988ce944877def2f44471d", size = 10411717, upload-time = "2026-07-22T22:18:08.307Z" }, + { url = "https://files.pythonhosted.org/packages/68/56/fbe81c09195924d8b7b8d4461a20458fe80a6a5ed6b24f0314da684277e1/pandas-3.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2e26bb46934b8a2ca0c3de1d3d606fc5f6746584791b2db264d58cf370e08dc", size = 9957095, upload-time = "2026-07-22T22:18:10.6Z" }, + { url = "https://files.pythonhosted.org/packages/e0/51/fac252f4a913ed5eabf3c11b880a9e8d5a6c10f0b2129d0462212d238b4d/pandas-3.0.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73fa87b08a7ef706f8aafda39ddaccf2a99047bea62d8c88a0361bcafb2237bc", size = 10485458, upload-time = "2026-07-22T22:18:12.834Z" }, + { url = "https://files.pythonhosted.org/packages/12/98/e976540c1addf70442be7842a18cf70884a964abbf69442504f4d2939989/pandas-3.0.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d373ce03ffd84010ed9839fa73672a9c8256990532e158440c0085db7d914b34", size = 10998091, upload-time = "2026-07-22T22:18:15.209Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8c/1f29b5be8d3fc47dd7567eb167fabba2085879b31e0287ce7cba6d3d2ff4/pandas-3.0.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a29c53d85ea98c5e792c59ef82ee9fbe6ca902c0d0adb6b23f45ef894cd7bf6", size = 11499501, upload-time = "2026-07-22T22:18:17.689Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e2/bd9c98ad2df7b38bde002adde4cdf353519da51881634323b126c55997f9/pandas-3.0.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ad3b02ed6bc7d7ae9b70804b2c6aa31827489d150f8e623ce82491b82085d7", size = 12060559, upload-time = "2026-07-22T22:18:20.147Z" }, + { url = "https://files.pythonhosted.org/packages/f3/9a/ffbd852d58bd74a617fe2f8ee6a58a96982271ce41cf981eab22190b4a4b/pandas-3.0.5-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:b2acb4650527eec6822c3dadb2b771277b65e7dae7a267d4bccf65fd1bb3fbce", size = 7197652, upload-time = "2026-07-22T22:18:22.502Z" }, + { url = "https://files.pythonhosted.org/packages/70/b5/d2d3e9ae73362ba4229651b0ee1455cf78073a1ce585f6ff693782ce263e/pandas-3.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:80a611068e8a3ac23f7398c6c14eb46dc974e5cc9997f653e2dcfd1da74edd41", size = 9831691, upload-time = "2026-07-22T22:18:24.534Z" }, + { url = "https://files.pythonhosted.org/packages/52/51/dea1e89d6a6796b9c43f85a09b484ee03edb8a4c4842e73e200a8c11301c/pandas-3.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:25ff585b972a18ef1fe9ffa3ac6544d9950508aa76832e5147640b6022821e49", size = 9105796, upload-time = "2026-07-22T22:18:27.064Z" }, + { url = "https://files.pythonhosted.org/packages/bf/09/7b95c4a0025227d6f118c4039b423412ac6a982db02864166185d812fbc7/pandas-3.0.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c1c05a767fe8e5b4fe9e1c29806829c582052eaedb9120a3da83ba3f69e24a5b", size = 10385742, upload-time = "2026-07-22T22:18:29.346Z" }, + { url = "https://files.pythonhosted.org/packages/8d/0c/dc78fd8c4da477b4b5e8ad37295af352190d21ef63a9ee1bc071753074cc/pandas-3.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b86765f268b56f7e665b93bce9d5df69dee7f99e595cf8fb839483ab315942a3", size = 9932067, upload-time = "2026-07-22T22:18:31.833Z" }, + { url = "https://files.pythonhosted.org/packages/3e/71/3592c055cf44df9808550f9368ceda80ff2b224d355ef73fe251dcda1802/pandas-3.0.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c597ecf5616b5c420372c1d4d4c00dbbfba7398bea857dcc984347e1ea48417b", size = 10466756, upload-time = "2026-07-22T22:18:34.195Z" }, + { url = "https://files.pythonhosted.org/packages/e3/70/4363150359f95b4cb4bcbb34ca23572bb5495749a621a8f3d5a1ddfd293c/pandas-3.0.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b11c36e218331d0387cbe3a0a5f75162357a1d92d57b2b08a336ff94b19b2be", size = 10938525, upload-time = "2026-07-22T22:18:36.81Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d0/317e7a0c67c0e69fa905a0161409397a7dc2d46ff611f6ca4803352c042b/pandas-3.0.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf52e1f61d229496da17dc7ab54acdee627357e7008fd4fecba3d0ba2937fa58", size = 11489303, upload-time = "2026-07-22T22:18:39.287Z" }, + { url = "https://files.pythonhosted.org/packages/f1/8d/36dade89b49e4f9d5cbdbe863772581f98c0c6d78fc39ad4c557f6f2e17e/pandas-3.0.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db172144bb56422bd157812f3b021eacc255451470b31e2c633c349490a1cfee", size = 11989004, upload-time = "2026-07-22T22:18:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ba/18c4ec8a746e177da05a9e7a7963781d8ea195780724f854601b6ebd6b78/pandas-3.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:0d298e951f23016ce4699951d044ae6418dbc91bf68cefca0f77666fcbb4e5c6", size = 9826896, upload-time = "2026-07-22T22:18:44.539Z" }, + { url = "https://files.pythonhosted.org/packages/de/ec/28a57266b753799a87b8bc79e7887ac6fd981b8c6d2978a0b7e7b6bd708c/pandas-3.0.5-cp313-cp313-win_arm64.whl", hash = "sha256:66266d3442a5e8b3c90274c2b8b230bee42dd1c286bc822cc2f9f2c7e12b883e", size = 9094790, upload-time = "2026-07-22T22:18:47.468Z" }, + { url = "https://files.pythonhosted.org/packages/51/2f/cf6aae281264f4463f0875bcbb15fd2bb6d291cc535187dad1732475e4a9/pandas-3.0.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f264fc46911cc8131a7322a16199bbf8e353d27c10bb211f5bd0c814324dc36", size = 10390034, upload-time = "2026-07-22T22:18:49.818Z" }, + { url = "https://files.pythonhosted.org/packages/06/ec/5189518c7a7659c4bdcc6b1eb32c46c6f3c86b0661ffd84143d1112c7732/pandas-3.0.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:53730687fcd161883b24e10411c06d6a4c0f2275d2faf3bb2bc25deb4ba8007c", size = 9980065, upload-time = "2026-07-22T22:18:52.249Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f1/598503ce8d7e3c35601e0747ba288c7864baae66380725bc12f13f884dfe/pandas-3.0.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:960d3ebcf249f75206899fcd2c6de53f736b7265759ced0d3e559df0b8b709b0", size = 10545532, upload-time = "2026-07-22T22:18:54.813Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/ceae2adf7034e07e9910299fe412e1819c4f0dd520700a888bcb03625448/pandas-3.0.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e94c2c5ca43bd3ca32bf64d32308887b65e5f9bfd8023ea52755107a999f93b", size = 10963120, upload-time = "2026-07-22T22:18:57.42Z" }, + { url = "https://files.pythonhosted.org/packages/66/25/86e0f4451874eb79e688deeebe3c451fec4557f8952005818d800ee8ac7e/pandas-3.0.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e819dd5f62966b481a8cb649d3299ebd886a1ea91ed5a99bf7ce77c98d18ab94", size = 11563178, upload-time = "2026-07-22T22:18:59.729Z" }, + { url = "https://files.pythonhosted.org/packages/f3/45/8643daa3b4147e433adfcccefdd0380d3aad79d86b15d8999730fe1944d5/pandas-3.0.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c5ed2e7c06e91d340dfd091d7934f9bc82e4a36b95f647f090b9d1c9ac649da", size = 12028708, upload-time = "2026-07-22T22:19:02.164Z" }, + { url = "https://files.pythonhosted.org/packages/96/58/ad979ae617615576e8aafd569c9d4b62f1191d896e38f51d66ba06f3b89a/pandas-3.0.5-cp314-cp314-win_amd64.whl", hash = "sha256:cd8f7c6dc98527058ee6264219343f5392240a6f1bfa654fc5d79023020d0c92", size = 9951806, upload-time = "2026-07-22T22:19:04.596Z" }, + { url = "https://files.pythonhosted.org/packages/69/32/7ac03886b304049a9d2625ee88f59af760d8a93bd30ed9239bce7b9869a8/pandas-3.0.5-cp314-cp314-win_arm64.whl", hash = "sha256:5183427f5a8156d480f30333777bc978be93650a49a7c01db26adffe95b31e85", size = 9238297, upload-time = "2026-07-22T22:19:06.836Z" }, + { url = "https://files.pythonhosted.org/packages/be/ed/1d1f2ee5547d5167face2376d11c8b2a4c7bfff5a416ee7a9046891fab1e/pandas-3.0.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:303da736987d481074ca720ada325f8bd80c64ebc2d45ed79b29df3aaa4a26ca", size = 10849690, upload-time = "2026-07-22T22:19:09.391Z" }, + { url = "https://files.pythonhosted.org/packages/57/55/17e17152e98fbb0c4b1e562bc65387a2f20a80db0f4a86bf8d3a0e4248d4/pandas-3.0.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3b2801bbb049d0136f6c213eae02b5fca969384fc2064dd728d8620552aa49da", size = 10509945, upload-time = "2026-07-22T22:19:11.773Z" }, + { url = "https://files.pythonhosted.org/packages/88/90/817d44dbf83facf9556f33576d9af0a241981e7bb5c00606c0bcb5df8dda/pandas-3.0.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cce3a9d11d2b1f82c69a27ec1f4948a170e2c403c4bbfa8cca62e3fdebe2ef3a", size = 10392197, upload-time = "2026-07-22T22:19:14.024Z" }, + { url = "https://files.pythonhosted.org/packages/f1/da/889f00c0a6f5aa1545add70abbf01502dff87ab577adb855bd631c54d2f2/pandas-3.0.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef01af4d8dc6cd2c8d6c7736f149574ef93fe043811eeb5e445f2647154b5040", size = 10862726, upload-time = "2026-07-22T22:19:16.351Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/f1e934fb3c98fce859c6147c6785816c7b5b9ab7821115c5d8c4de9842b9/pandas-3.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2759e890db96dfcffdbd9b86c3c2cb6afaf58def482820317e06163ec1066cd", size = 11414864, upload-time = "2026-07-22T22:19:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/fe/be/d448af7d657d82e1888dd8551f79c6d6fb161080b5b9752d84d910ec2319/pandas-3.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b58b1b39d46a5862e3fb18f50d1a201398619d16a0f9f73f57eea5583cf0e63c", size = 11925105, upload-time = "2026-07-22T22:19:21.515Z" }, + { url = "https://files.pythonhosted.org/packages/29/c1/ccb4238212c8c4f496c584f3044d94e0c030ed8e1d68999db46c91c2242f/pandas-3.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:1c10461f6eeb35d8f05b6184c65c8b9991663b66c46b1d559b682cb34ae7c6ea", size = 10387612, upload-time = "2026-07-22T22:19:24.257Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cf/6a51b2c38980e04c279fd2fa908a1b0982064e860444acfca4ec2e2c8359/pandas-3.0.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3c5015fd1730fbf883647e88068176c839c102cea883ba1769a6f4593bfc1f8c", size = 9509776, upload-time = "2026-07-22T22:19:26.694Z" }, +] + +[[package]] +name = "parso" +version = "0.8.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/30/4b/90c937815137d43ce71ba043cd3566221e9df6b9c805f24b5d138c9d40a7/parso-0.8.7.tar.gz", hash = "sha256:eaaac4c9fdd5e9e8852dc778d2d7405897ec510f2a298071453e5e3a07914bb1", size = 401824, upload-time = "2026-05-01T23:13:02.138Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl", hash = "sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c", size = 107025, upload-time = "2026-05-01T23:12:58.867Z" }, +] + +[[package]] +name = "pexpect" +version = "4.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ptyprocess" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, +] + +[[package]] +name = "pillow" +version = "12.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/bf/fb3ebff8ddcb76aac5a01389251bbbb9519922a9b520d8247c1ca864a25d/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965", size = 5345969, upload-time = "2026-07-01T11:54:06.397Z" }, + { url = "https://files.pythonhosted.org/packages/d8/66/9a386a92561f402389a4fc70c18838bf6d35eb5eb5c6850b4b2dc64f5048/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7", size = 4780323, upload-time = "2026-07-01T11:54:09.351Z" }, + { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, + { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, + { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, + { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, + { url = "https://files.pythonhosted.org/packages/20/20/25e0f4dc178a6bc0696793720055519a0de89e7661dae886992decbd2f81/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f", size = 6472684, upload-time = "2026-07-01T11:54:19.839Z" }, + { url = "https://files.pythonhosted.org/packages/45/89/da2f7971a317f83d807fdd4065c0af40208e59e692cc43d315a71a0e96d1/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09", size = 7227137, upload-time = "2026-07-01T11:54:22.025Z" }, + { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ac/31fb64e1e7efb5a4b50cd3d92049ba89ac6e4d8d3bb6a74e15048ca3353e/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89", size = 4161684, upload-time = "2026-07-01T11:54:25.934Z" }, + { url = "https://files.pythonhosted.org/packages/87/b4/9805e23d2b4d77842b468513841fda254ee42f0289d25088340e4ff46e2d/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace", size = 4255487, upload-time = "2026-07-01T11:54:27.935Z" }, + { url = "https://files.pythonhosted.org/packages/df/39/ecf519435a200c693fe053a6ee4d835b41cf963a4dfc2551c4e637cb2a71/pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec", size = 3696433, upload-time = "2026-07-01T11:54:29.813Z" }, + { url = "https://files.pythonhosted.org/packages/42/92/2fc3ffad878ae8dd5469ec1bc8eb83b71f48e13efdf68f02709003982a32/pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66", size = 5345889, upload-time = "2026-07-01T11:54:31.97Z" }, + { url = "https://files.pythonhosted.org/packages/10/76/8803c13605b763d33d156c4678fc77f8443389c0c51c8aef707bb02015f4/pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35", size = 4780109, upload-time = "2026-07-01T11:54:34.026Z" }, + { url = "https://files.pythonhosted.org/packages/1f/01/e18aff37cb0b4aac47ac90f016d347a49aca667ef97f190b06ac2aabc928/pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65", size = 6263736, upload-time = "2026-07-01T11:54:36.131Z" }, + { url = "https://files.pythonhosted.org/packages/f7/62/de5bdd77d935331f4f802edc11e4d82950f642caad6cb2f949837b8560e2/pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3", size = 6937129, upload-time = "2026-07-01T11:54:38.216Z" }, + { url = "https://files.pythonhosted.org/packages/70/4d/105627a13300c5e0df1d174230b32fd1273062c96f7745fd552b945d1e1d/pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a", size = 6339562, upload-time = "2026-07-01T11:54:40.354Z" }, + { url = "https://files.pythonhosted.org/packages/6b/1d/f13de01a553988ab895ba1c722e06cf3144d4f57656fd5b81b6d881f1179/pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e", size = 7049439, upload-time = "2026-07-01T11:54:42.489Z" }, + { url = "https://files.pythonhosted.org/packages/c9/f9/066794cca041b969964f779ee5fa66a9498bbf34248ac39c5d7954e4198f/pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f", size = 6473287, upload-time = "2026-07-01T11:54:44.9Z" }, + { url = "https://files.pythonhosted.org/packages/a6/9b/7a58e61d62be561da3a356fe2384d4059a6345fc130e23ef1c36a5b81d24/pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8", size = 7239691, upload-time = "2026-07-01T11:54:47.141Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b0/c4ed4f0ef8f8fa5ee8351537db6650bb8189f7e118842978dd6589065692/pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b", size = 2568185, upload-time = "2026-07-01T11:54:49.137Z" }, + { url = "https://files.pythonhosted.org/packages/dc/01/001f65b68192f0228cc1dbbc8d2530ab5d58b61037ba0587f946fea607cd/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330", size = 4161736, upload-time = "2026-07-01T11:54:51.156Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d2/0219746d0fd16fc8a84498e79452375be3797d3ce4044596ce565164b84f/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217", size = 4255435, upload-time = "2026-07-01T11:54:53.414Z" }, + { url = "https://files.pythonhosted.org/packages/c8/02/8d0bc62ef0302318c46ff2a512822d2610e81c7aa46c9b3abe6cbaca5ad0/pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930", size = 3696262, upload-time = "2026-07-01T11:54:55.739Z" }, + { url = "https://files.pythonhosted.org/packages/85/e2/73c77d218410b14f5f2d565e8a998d5317b7b9c75368d29985139f7a46f0/pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8", size = 5350344, upload-time = "2026-07-01T11:54:57.657Z" }, + { url = "https://files.pythonhosted.org/packages/c7/da/32c752228ae345f489e3a42499d817b6c3996da7e8a3bc7a04fc806b243b/pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0", size = 4780131, upload-time = "2026-07-01T11:54:59.713Z" }, + { url = "https://files.pythonhosted.org/packages/b1/9d/8b2c807dbef61a5197c047afe99823787eb66f63daf9fb2432f91d6f0462/pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321", size = 6263757, upload-time = "2026-07-01T11:55:01.778Z" }, + { url = "https://files.pythonhosted.org/packages/5c/44/c85361f65dbe00eea8576ee467c768d25129989efb76e94f205e9ca9bb46/pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b", size = 6936962, upload-time = "2026-07-01T11:55:03.93Z" }, + { url = "https://files.pythonhosted.org/packages/18/7e/e483414b35800b86b6f08dbbc7803fb5cd52c4d6f897f47d53ea2c7e6f65/pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198", size = 6339171, upload-time = "2026-07-01T11:55:05.989Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f4/68c491844841ede6bed70189546b3ee9731cf9f2cbad396faff5e1ccba45/pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130", size = 7048116, upload-time = "2026-07-01T11:55:08.131Z" }, + { url = "https://files.pythonhosted.org/packages/a3/34/77f3f793fed8efc7d243f21b33c5a3f0d1c97ee70346d3db855587e155ff/pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a", size = 6467209, upload-time = "2026-07-01T11:55:10.408Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e0/492879f69d94f91f60fc8cd05ba03650e9520afebb2fb7aa12777d7c7f38/pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d", size = 7237707, upload-time = "2026-07-01T11:55:12.745Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ac/6b11f2875f1c2ac040d84e1bbf9cf22a88038f901ca1037898b280b38365/pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838", size = 2565995, upload-time = "2026-07-01T11:55:14.736Z" }, + { url = "https://files.pythonhosted.org/packages/52/69/c2208e56af9bfc1913afb24020297a691eb1d4ef688474c8a04913f65e04/pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e", size = 5352503, upload-time = "2026-07-01T11:55:17.076Z" }, + { url = "https://files.pythonhosted.org/packages/07/70/e5686d753e898a45d778ff1718dba8516ead6ab6b95d85fc8c4b70650cf2/pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17", size = 4782956, upload-time = "2026-07-01T11:55:19.448Z" }, + { url = "https://files.pythonhosted.org/packages/d5/37/25c6692f06927ee973ff18c8d9ee98ad0b4d84ee67a09610c2dd1447958e/pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385", size = 6322855, upload-time = "2026-07-01T11:55:21.613Z" }, + { url = "https://files.pythonhosted.org/packages/cc/91/420637fcb8f1bc11029e403b4538e6694744428d8246118e45719f944556/pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c", size = 6989642, upload-time = "2026-07-01T11:55:24.006Z" }, + { url = "https://files.pythonhosted.org/packages/10/08/b94d7811281ccf0d143a1cf768d1c49e1e54af63e7b708ab2ee3eb87face/pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d", size = 6391281, upload-time = "2026-07-01T11:55:26.252Z" }, + { url = "https://files.pythonhosted.org/packages/d2/87/24233f785f55474dc02ce3e739c5528a77e3a862e9333d1dd7a25cc31f70/pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931", size = 7096716, upload-time = "2026-07-01T11:55:28.318Z" }, + { url = "https://files.pythonhosted.org/packages/23/26/fcb2f6e37175b04f53570b59937867e2b80ee1685e744023153028fc14f9/pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7", size = 6474125, upload-time = "2026-07-01T11:55:30.956Z" }, + { url = "https://files.pythonhosted.org/packages/90/de/3634abee5f1c9e13c56787b7d5517b0ba8d6de51700b95578cf338349c9f/pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c", size = 7242939, upload-time = "2026-07-01T11:55:34.044Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2a/fd13f8eb24de5714a6eb444a3d67e2842c6c576e159a43793adf23051351/pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45", size = 2567506, upload-time = "2026-07-01T11:55:35.988Z" }, + { url = "https://files.pythonhosted.org/packages/5d/dc/8fdce34ec725a33c81c6ba122b904d6b9024e50ea9ac7bede62fab54506c/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139", size = 4162063, upload-time = "2026-07-01T11:55:37.941Z" }, + { url = "https://files.pythonhosted.org/packages/76/66/2044b9a63d3b84ff048228dfcb7cd9bf0df983e8470971bf7d4c57b693de/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402", size = 4255549, upload-time = "2026-07-01T11:55:40.022Z" }, + { url = "https://files.pythonhosted.org/packages/52/7e/1f67e6f4ece6b582ee4b539decbcc9f848dc245a93ed8cd7338bafef72f1/pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c", size = 3696331, upload-time = "2026-07-01T11:55:41.98Z" }, + { url = "https://files.pythonhosted.org/packages/12/40/d306fc2c8e4d45d7f175c77edca7063be7b86fe7fe6e68f4353bf71d808c/pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f", size = 5350370, upload-time = "2026-07-01T11:55:44.028Z" }, + { url = "https://files.pythonhosted.org/packages/dd/44/668fb1437e8ce420f62d6106eb66e44a5971602a4d794615bdf79315d82d/pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701", size = 4780147, upload-time = "2026-07-01T11:55:46.073Z" }, + { url = "https://files.pythonhosted.org/packages/0c/08/93fa2e70e30a2d81547e481b6ee2bb9522117221fb1e0ce4b5df70967677/pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace", size = 6273659, upload-time = "2026-07-01T11:55:48.264Z" }, + { url = "https://files.pythonhosted.org/packages/f8/6d/043e96ff814fc31a33077e4cba86082167db520c93632afdf2042febbb0c/pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4", size = 6947439, upload-time = "2026-07-01T11:55:50.503Z" }, + { url = "https://files.pythonhosted.org/packages/af/92/ba71d2ee2ac0edf3fa33bd9d5ee9ee080da70b1766f3ca3934f9938ddac9/pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39", size = 6353577, upload-time = "2026-07-01T11:55:52.697Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ce/e63064e2122923ff687c8ad792d0d736a7b3920a56a46982e81a7fdd25d6/pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71", size = 7060394, upload-time = "2026-07-01T11:55:55.149Z" }, + { url = "https://files.pythonhosted.org/packages/54/76/a09cc3ccc8d773a7283d34c38bec1708f9e3cc932093cbc4c5e71ac4060b/pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827", size = 6467375, upload-time = "2026-07-01T11:55:57.769Z" }, + { url = "https://files.pythonhosted.org/packages/3e/03/1846c49ba3b1d5550392a4bbd06d6fb4578e1cd91a803198b5c90f5f7d53/pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5", size = 7237048, upload-time = "2026-07-01T11:55:59.975Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bb/89f35dcc79610423f9f195504d7def7f0d1416a711541b42867e25fe3412/pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658", size = 2566006, upload-time = "2026-07-01T11:56:02.143Z" }, + { url = "https://files.pythonhosted.org/packages/30/88/707027ba09942dfa2c28759b5c222d769290a41c6d20ea60ec250801941f/pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf", size = 5352509, upload-time = "2026-07-01T11:56:04.2Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6d/00352fa25332c2569cd387851f568cc5a4b75a9adbfb37ac4fbce4c02eec/pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64", size = 4783167, upload-time = "2026-07-01T11:56:06.631Z" }, + { url = "https://files.pythonhosted.org/packages/13/4f/9e049dfa21af7c22427275720e2490267ba8138120add5c4c574deb69782/pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e", size = 6329237, upload-time = "2026-07-01T11:56:08.868Z" }, + { url = "https://files.pythonhosted.org/packages/36/16/cf6eeaae8d0fce8dd390a33437cf68c5d5bd73834a2bc6e2f14efda0ab45/pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777", size = 6997047, upload-time = "2026-07-01T11:56:11.379Z" }, + { url = "https://files.pythonhosted.org/packages/1e/69/dbf769bdd55f48bf5733cac28edc6364ffaa072ec9ba336266e4fe66be55/pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1", size = 6400440, upload-time = "2026-07-01T11:56:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, + { url = "https://files.pythonhosted.org/packages/18/f0/a5595c1e8c3ae44b9828cb2f0fa8155e5095ef04d6327b8f61cf44a3df85/pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8", size = 6474384, upload-time = "2026-07-01T11:56:18.855Z" }, + { url = "https://files.pythonhosted.org/packages/e4/04/62bcd9f844984c5938d3b05264a61d797a29d3e0812341a8204af70bbdee/pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418", size = 7243537, upload-time = "2026-07-01T11:56:21.214Z" }, + { url = "https://files.pythonhosted.org/packages/3d/68/1f3066acedf37673694a7141381d8f811ae97f30d34413d236abe7d489f1/pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59", size = 2567491, upload-time = "2026-07-01T11:56:23.506Z" }, +] + +[[package]] +name = "playwright" +version = "1.62.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/5b/ca2abcf3aa69f9fb510215e3064f30b57fe57657c8d04ede45bb966d5606/playwright-1.62.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:d8da938f3748841a8754f2e1f0216902c1c8f8ae3720de8b32ccf8e6913a7c4f", size = 43732091, upload-time = "2026-07-31T17:00:44.178Z" }, + { url = "https://files.pythonhosted.org/packages/af/1a/0bfbe9904350961f4dbb713f04342e40d548c5fc26c8157bd13617c81492/playwright-1.62.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:db755ab27db21a04186f1fe8169888e42356086e439b1059b923ef417f0b6034", size = 42510842, upload-time = "2026-07-31T17:00:48.596Z" }, + { url = "https://files.pythonhosted.org/packages/66/dc/c0486b407ad0699a250f6bbe3066fca95344009a99ca66e88ca175c69dc1/playwright-1.62.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:5108bd5b3e87169ddf269feee097da5893af7f8aea4634dfc840518d64c1f1da", size = 43732093, upload-time = "2026-07-31T17:00:52.218Z" }, + { url = "https://files.pythonhosted.org/packages/43/6b/b24aebc2b04bffcb342bccf96e287c78b363e1615bed5cea97500cc0393a/playwright-1.62.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:ba33bae6a13b3d9d354c751cb618af357d20fe1d57767cbcce52079bbef17ad3", size = 47748926, upload-time = "2026-07-31T17:00:56.438Z" }, + { url = "https://files.pythonhosted.org/packages/36/43/b4b18bdc87e1949568fffdcde3ff9a0456266b2d0c6d4432cc34d89ea6eb/playwright-1.62.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db2d76613a57ad844362ce42f7d0c2fa26b19a4f7a46d4f76b891c631e6e5aff", size = 47441423, upload-time = "2026-07-31T17:01:00.404Z" }, + { url = "https://files.pythonhosted.org/packages/81/22/af5d926fc2c32a339eec00a443644bc40ab9db1dd2dd9017873c59773c0c/playwright-1.62.0-py3-none-win32.whl", hash = "sha256:e5614fa89355d7081457680324bb219f79f69c423c5cb6fa250e30b0d8aebf1c", size = 38164450, upload-time = "2026-07-31T17:01:04.187Z" }, + { url = "https://files.pythonhosted.org/packages/2b/a9/4160c1033c07af98bf841ad079457dd78408a5ee0dd56cbfe50b8b6a1c22/playwright-1.62.0-py3-none-win_amd64.whl", hash = "sha256:92c0d98ed04eb35af557b709875edba415b1f548bdb22ddb5bb3e1e6c835c2f1", size = 38164458, upload-time = "2026-07-31T17:01:08.459Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ec/06b55d619a7082a766aa04f2c6bb31435c87f02930087d8a0517119408fa/playwright-1.62.0-py3-none-win_arm64.whl", hash = "sha256:ea8d3055aa9d5a9f1832ac82517bd8b42c78fac7ebcbebb0107116735c8cb6a1", size = 34208868, upload-time = "2026-07-31T17:01:11.818Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "plum-dispatch" +version = "1.7.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/39/4e/f4d6b2bd80a9880989d37773f0f2e3a91a2d5352732e4ebb68b5606bca83/plum-dispatch-1.7.4.tar.gz", hash = "sha256:1c1d15b2842b5fa98405fd3dff6fad4887bdc77b60bd200e209d76ebfe9990fe", size = 56429, upload-time = "2022-10-21T06:29:20.747Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/b6/3aaa985591c63da64c7bd8c5f470442a4c00b37ad3ed057f21de14174f83/plum_dispatch-1.7.4-py3-none-any.whl", hash = "sha256:c40dbeab269bbbf972ce0dbc078380da19ebaee1a370a2c564e1814a11bde216", size = 24238, upload-time = "2022-10-21T06:29:19.054Z" }, +] + +[[package]] +name = "polygon3" +version = "3.0.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1f/26/eea4112be43c8b7345477ad9150d499303494f32fb5951cb0f6e9104045b/Polygon3-3.0.9.1.tar.gz", hash = "sha256:2ddf8d06975f728d5b40786136c82e5b9d38a846bce236b7e6587bbd6a5e9b49", size = 39121, upload-time = "2021-03-09T16:04:50.975Z" } + +[[package]] +name = "prompt-toolkit" +version = "3.0.53" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/ea/39b988c938f75cb75d7045b5c69f8bfed47ee2152c8837fb403de29d6fb8/prompt_toolkit-3.0.53.tar.gz", hash = "sha256:9ec8a0ad96d5c56148b3f914aa79c1564c3fde5d2e6b876e7bc327e353cf8fa6", size = 435492, upload-time = "2026-07-26T20:56:14.758Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/6f/84908cad2d6aa5144abcf7b42709fe4fdb459bc640ec7ac5786e7693dabc/prompt_toolkit-3.0.53-py3-none-any.whl", hash = "sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2", size = 392288, upload-time = "2026-07-26T20:56:12.512Z" }, +] + +[[package]] +name = "propcache" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/cb/e27bc2b2737a0bb49962b275efa051e8f1c35a936df7d5139b6b658b7dc9/propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba", size = 95887, upload-time = "2026-05-08T21:00:11.277Z" }, + { url = "https://files.pythonhosted.org/packages/e6/13/b8ae04c59392f8d11c6cd9fb4011d1dc7c86b81225c770280300e259ffe1/propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a", size = 54654, upload-time = "2026-05-08T21:00:12.604Z" }, + { url = "https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf", size = 55190, upload-time = "2026-05-08T21:00:13.935Z" }, + { url = "https://files.pythonhosted.org/packages/44/c7/085d0cd63062e84044e3f05797749c3f8e3938ff3aeb0eb2f69d43fafc91/propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144", size = 59995, upload-time = "2026-05-08T21:00:15.526Z" }, + { url = "https://files.pythonhosted.org/packages/9c/42/32cf8e3009e92b2645cf1e944f701e8ea4e924dffde1ee26db860bcbf7e4/propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9", size = 63422, upload-time = "2026-05-08T21:00:16.824Z" }, + { url = "https://files.pythonhosted.org/packages/9e/1b/f112433f99fc979431b87a39ef169e3f8df070d99a72792c56d6937ac48b/propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42", size = 64342, upload-time = "2026-05-08T21:00:18.362Z" }, + { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639, upload-time = "2026-05-08T21:00:19.692Z" }, + { url = "https://files.pythonhosted.org/packages/cc/da/4d775080b1490c0ae604acda868bd71aabe3a89ed16f2aa4339eb8a283e7/propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba", size = 61588, upload-time = "2026-05-08T21:00:21.155Z" }, + { url = "https://files.pythonhosted.org/packages/04/ac/f076982cbe2195ee9cf32de5a1e46951d9fb399fc207f390562dd0fd8fb2/propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a", size = 60029, upload-time = "2026-05-08T21:00:22.713Z" }, + { url = "https://files.pythonhosted.org/packages/70/60/189be62e0dd898dce3b331e1b8c7a543cd3a405ac0c81fe8ee8a9d5d77e1/propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64", size = 56774, upload-time = "2026-05-08T21:00:24.001Z" }, + { url = "https://files.pythonhosted.org/packages/ea/9e/93377b9c7939c1ffae98f878dee955efadfd638078bc86dbc21f9d52f651/propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913", size = 63532, upload-time = "2026-05-08T21:00:25.545Z" }, + { url = "https://files.pythonhosted.org/packages/14/f9/590ef6cfb9b8028d516d287812ece32bb0bc5f11fbb9c8bf6b2e6313fec8/propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1", size = 61592, upload-time = "2026-05-08T21:00:27.186Z" }, + { url = "https://files.pythonhosted.org/packages/b4/5e/70958b3034c297a630bba2f17ca7abc2d5f39a803ad7e370ab79d1ecd022/propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33", size = 64788, upload-time = "2026-05-08T21:00:28.8Z" }, + { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514, upload-time = "2026-05-08T21:00:30.098Z" }, + { url = "https://files.pythonhosted.org/packages/cf/74/66bd798b5b3be70aa1b391f5cc9d6a0a5532d7fd3b19ec0b213e72e6ad9d/propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031", size = 39018, upload-time = "2026-05-08T21:00:31.622Z" }, + { url = "https://files.pythonhosted.org/packages/61/7c/5c0d34aa3024694d6dcb9271cdbdd08c4e47c1c0ad95ec7e7bc74cdea145/propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42", size = 42322, upload-time = "2026-05-08T21:00:32.918Z" }, + { url = "https://files.pythonhosted.org/packages/4d/91/875812f1a3feb20ceba818ef39fbe4d92f1081e04ac815c822496d0d038b/propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84", size = 38172, upload-time = "2026-05-08T21:00:35.124Z" }, + { url = "https://files.pythonhosted.org/packages/c5/09/f049e45385503fe67db75a6b6186a7b9f0c3930366dc960522c312a825b1/propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a", size = 94457, upload-time = "2026-05-08T21:00:36.355Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/83d1d05655baf63113731bd5a1008435e14f8d1e5a06cbe4ec5b23ad7a31/propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117", size = 53835, upload-time = "2026-05-08T21:00:38.072Z" }, + { url = "https://files.pythonhosted.org/packages/a9/12/a6ba6482bb5ea3260c000c9b20881c95fa11c6b30173715668259f844ed7/propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098", size = 54545, upload-time = "2026-05-08T21:00:39.319Z" }, + { url = "https://files.pythonhosted.org/packages/a9/19/7fa086f5764c59ec8a8e157cd93aa8497acc00aba9dcdec56bfffb32602d/propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4", size = 59886, upload-time = "2026-05-08T21:00:40.621Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e4/5d7663dc8235956c8f5281698a3af1d351d8820341ddd890f59d9a9127f2/propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e", size = 63261, upload-time = "2026-05-08T21:00:41.775Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4a/15a03adee24d6350da4292caeac44c34c033d2afe5e87eb370f38854560f/propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7", size = 64184, upload-time = "2026-05-08T21:00:43.018Z" }, + { url = "https://files.pythonhosted.org/packages/8b/c6/979176efdaa3d239e36d503d5af63a0a773b36662ed8f52e5b6a6d9fd40e/propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d", size = 61534, upload-time = "2026-05-08T21:00:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/c8/22/63e8cd1bae4c2d2be6493b6b7d10566ddafad88137cfbc99964a1119853c/propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a", size = 61500, upload-time = "2026-05-08T21:00:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/60/5a/28e5d9acbac1cc9ccb67045e8c1b943aa8d79fdf39c93bd73cacd68008ea/propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2", size = 59994, upload-time = "2026-05-08T21:00:47.093Z" }, + { url = "https://files.pythonhosted.org/packages/f3/40/db650677f554a95b9c01a7c9d93d629e93a15562f5deb4573c9ee136fed2/propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa", size = 56884, upload-time = "2026-05-08T21:00:48.376Z" }, + { url = "https://files.pythonhosted.org/packages/80/45/70b39b89516ff8b96bf732fa6fded8cef20f293cb1508690101c3c07ec51/propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853", size = 63464, upload-time = "2026-05-08T21:00:49.954Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e2/fa59d3a89eac5534293124af4f1d0d0ada091ce4a0ab4610ce03fd2bdd8d/propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a", size = 61588, upload-time = "2026-05-08T21:00:51.281Z" }, + { url = "https://files.pythonhosted.org/packages/0b/97/efb547a55c4bc7381cfb202d6a2239ac621045277bc1ea5dfd3a7f0516c0/propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704", size = 64667, upload-time = "2026-05-08T21:00:52.602Z" }, + { url = "https://files.pythonhosted.org/packages/92/56/f5c7d9b4b7595d5127da38974d791b2153f3d1eae6c674af3583ace92ad3/propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4", size = 62463, upload-time = "2026-05-08T21:00:54.303Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3b/484a3a65fc9f9f60c41dcd17b428bace5389544e2c680994534a20755066/propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d", size = 38621, upload-time = "2026-05-08T21:00:55.808Z" }, + { url = "https://files.pythonhosted.org/packages/1c/fd/3f0f10dba4dabad3bf53102be007abf55481067952bde0fdddff439e7c61/propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757", size = 41649, upload-time = "2026-05-08T21:00:57.061Z" }, + { url = "https://files.pythonhosted.org/packages/90/ec/6ce619cc32bb500a482f811f9cd509368b4e58e638d13f2c68f370d6b475/propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f", size = 37636, upload-time = "2026-05-08T21:00:58.646Z" }, + { url = "https://files.pythonhosted.org/packages/1b/82/c1d268bbbf2ef981c5bf0fbbe746db617c66e3bcefe431a1aa8943fbe23a/propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d", size = 98872, upload-time = "2026-05-08T21:00:59.889Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d4/52c871e73e864e6b34c0e2d58ac1ec5ccd149497ddc7ad2137ae98323a35/propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa", size = 56257, upload-time = "2026-05-08T21:01:01.195Z" }, + { url = "https://files.pythonhosted.org/packages/67/f0/9b90ca2a210b3d09bcfcd96ecd0f55545c091535abce2a45de2775cfd357/propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94", size = 56696, upload-time = "2026-05-08T21:01:02.941Z" }, + { url = "https://files.pythonhosted.org/packages/9d/0e/6e9d4ba07c8e56e21ddec1e75f12148142b21ca83a51871babce095334f4/propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164", size = 62378, upload-time = "2026-05-08T21:01:04.475Z" }, + { url = "https://files.pythonhosted.org/packages/65/19/c10badaa463dde8a27ce884f8ee2ec37e6035b7c9f5ff0c8f74f06f08dac/propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f", size = 65283, upload-time = "2026-05-08T21:01:05.959Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b6/93bea99ca80e19cef6512a8580e5b7857bbe09422d9daa7fd4ef5723306c/propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c", size = 66616, upload-time = "2026-05-08T21:01:07.228Z" }, + { url = "https://files.pythonhosted.org/packages/83/e4/5c7462e50625f051f37fb38b8224f7639f667184bbd34424ec83819bb1b7/propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc", size = 63773, upload-time = "2026-05-08T21:01:08.514Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b6/99238894047b13c823be25027e736626cd414a52a5e30d2c3347c2733529/propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f", size = 63664, upload-time = "2026-05-08T21:01:09.874Z" }, + { url = "https://files.pythonhosted.org/packages/85/1e/a3a1a63116a2b8edb415a8bb9a6f0c34bd03830b1e18e8ce2904e1dc1cf4/propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb", size = 62643, upload-time = "2026-05-08T21:01:11.132Z" }, + { url = "https://files.pythonhosted.org/packages/e4/03/893cf147de2fc6543c5eaa07ad833170e7e2a2385725bbebe8c0503723bb/propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751", size = 59595, upload-time = "2026-05-08T21:01:12.387Z" }, + { url = "https://files.pythonhosted.org/packages/86/3b/04c1a2e12c57766568ba75ba72b3bf2042818d4c1425fab6fc07155c7cff/propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836", size = 65711, upload-time = "2026-05-08T21:01:13.676Z" }, + { url = "https://files.pythonhosted.org/packages/1c/34/80f8d0099f8d6bacc4de1624c85672681c8cd1149ca2da0e38fd120b817f/propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f", size = 64247, upload-time = "2026-05-08T21:01:14.936Z" }, + { url = "https://files.pythonhosted.org/packages/f3/1a/8b08f3a5f1037e9e370c55883ceeeee0f6dd0416fb2d2d67b8bfc91f2a79/propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55", size = 67102, upload-time = "2026-05-08T21:01:16.281Z" }, + { url = "https://files.pythonhosted.org/packages/34/68/8bdb7bb7756d76e005490649d10e4a8369e610c74d619f71e1aedf889e9c/propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568", size = 64964, upload-time = "2026-05-08T21:01:17.57Z" }, + { url = "https://files.pythonhosted.org/packages/0a/aa/50fb0b5d3968b61a510926ff8b8465f1d6e976b3ab74496d7a4b9fc42515/propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191", size = 42546, upload-time = "2026-05-08T21:01:18.946Z" }, + { url = "https://files.pythonhosted.org/packages/ae/4c/0ddbae64321bd4a95bcbfc19307238016b5b1fee645c84626c8d539e5b74/propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7", size = 46330, upload-time = "2026-05-08T21:01:20.162Z" }, + { url = "https://files.pythonhosted.org/packages/00/d9/9cddc8efb78d8af264c5ec9f6d10b62f57c515feda8d321595f56010fb23/propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96", size = 40521, upload-time = "2026-05-08T21:01:21.399Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ea/23ee535d90ce8bcc465a3028eb3cc0ce3bd1005f4bb27710b30587de798d/propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999", size = 94662, upload-time = "2026-05-08T21:01:22.683Z" }, + { url = "https://files.pythonhosted.org/packages/b5/06/c5a52f419b5d8972f8d46a7577476090d8e3263ff589ce40b5ca4968d5be/propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e", size = 53928, upload-time = "2026-05-08T21:01:23.986Z" }, + { url = "https://files.pythonhosted.org/packages/63/b1/4260d67d6bd85e58a66b72d54ce15d5de789b6f3870cc6bedf8ff9667401/propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539", size = 54650, upload-time = "2026-05-08T21:01:25.305Z" }, + { url = "https://files.pythonhosted.org/packages/70/06/2f46c318e3307cd7a6a7481def374ce838c0fe20084b39dd54b0879d0e99/propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e", size = 59912, upload-time = "2026-05-08T21:01:26.545Z" }, + { url = "https://files.pythonhosted.org/packages/4c/29/fe1aebec2ce57ab985a9c382bded1124431f85078113aa222c5d278430d4/propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979", size = 63300, upload-time = "2026-05-08T21:01:27.937Z" }, + { url = "https://files.pythonhosted.org/packages/b4/18/2334b26768b6c82be8c69e83671b767d5ef426aa09b0cba6c2ea47816774/propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80", size = 64208, upload-time = "2026-05-08T21:01:29.484Z" }, + { url = "https://files.pythonhosted.org/packages/2b/76/7f1bfd6afff4c5e38e36a3c6d68eb5f4b7311ea80baf693db78d95b603c4/propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825", size = 61633, upload-time = "2026-05-08T21:01:31.068Z" }, + { url = "https://files.pythonhosted.org/packages/c4/46/b3ff8aba2b4953a3e50de2cf72f1b5748b8eca93b15f3dc2c84339084c09/propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39", size = 61724, upload-time = "2026-05-08T21:01:32.374Z" }, + { url = "https://files.pythonhosted.org/packages/c5/01/814cfcafbcff954f94c01cf30e097ddc88a076b5440fbcf4570753437d40/propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4", size = 60069, upload-time = "2026-05-08T21:01:33.67Z" }, + { url = "https://files.pythonhosted.org/packages/da/68/5c6f7622d510cc666a300687e06fd060c1a43361c0c9b20d284f06d8096a/propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5", size = 57099, upload-time = "2026-05-08T21:01:34.915Z" }, + { url = "https://files.pythonhosted.org/packages/55/27/9cb0b4c679124085327957d42521c99dba04c88c90c3e55a6f0b633ebccc/propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702", size = 63391, upload-time = "2026-05-08T21:01:36.231Z" }, + { url = "https://files.pythonhosted.org/packages/f0/9d/7258aaa5bdf60fc6f27591eef6fe52768cb0beda7140be477c8b12c9794a/propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3", size = 61626, upload-time = "2026-05-08T21:01:37.545Z" }, + { url = "https://files.pythonhosted.org/packages/8e/0d/41c602003e8a9b16fe1e7eadf62c7bfba9d5474370b24200bf48b315f45f/propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5", size = 64781, upload-time = "2026-05-08T21:01:38.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f3/38e66b1856e9bd079deea015bc4a55f7767c0e4db2f7dcf69e7e680ba4ce/propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4", size = 62570, upload-time = "2026-05-08T21:01:40.415Z" }, + { url = "https://files.pythonhosted.org/packages/95/ca/bbfe9b910ce57dde8bb4876b4520fc02a4e89497c10de26be936758a3aaa/propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0", size = 39436, upload-time = "2026-05-08T21:01:41.654Z" }, + { url = "https://files.pythonhosted.org/packages/61/d2/45c9defbaa1ea297035d9d4cce9e8f80daafbf19319c6007f157c6256ea9/propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c", size = 42373, upload-time = "2026-05-08T21:01:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/44/68/9ea5103f41d5217d7d6ec24db90018e23aebec070c3f9a6e54d12b841fd8/propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0", size = 38554, upload-time = "2026-05-08T21:01:44.336Z" }, + { url = "https://files.pythonhosted.org/packages/8a/81/fadf555f42d3b762eea8a53950b0489fdc0aa9da5f8ed9e10ce0a4e01b48/propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb", size = 99395, upload-time = "2026-05-08T21:01:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/f5/c9/c61e134a686949cf7971af3a390148b1156f7be81c73bc0cd12c873e2d48/propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078", size = 56653, upload-time = "2026-05-08T21:01:47.307Z" }, + { url = "https://files.pythonhosted.org/packages/cb/73/daf935ea7048ddd7ec8eec5345b4a40b619d2d178b3c0a0900796bc3c794/propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa", size = 56914, upload-time = "2026-05-08T21:01:48.573Z" }, + { url = "https://files.pythonhosted.org/packages/79/9f/aba959b435ea18617edd7cf0a7ad0b9c574b8fc7e3d2cd55fb59cb255d33/propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917", size = 62567, upload-time = "2026-05-08T21:01:49.903Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a1/859942de9a791ff42f6141736f5b37749b8f53e65edfa49638c67dd67e6a/propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe", size = 65542, upload-time = "2026-05-08T21:01:51.204Z" }, + { url = "https://files.pythonhosted.org/packages/b5/61/315bc0fd6c0fc7f80a528b8afd209e5fc4a875ea79571b91b8f50f442907/propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03", size = 66845, upload-time = "2026-05-08T21:01:52.539Z" }, + { url = "https://files.pythonhosted.org/packages/47/f7/9f8122e3132e8e354ac41975ef8f1099be7d5a16bc7ae562734e993665c0/propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335", size = 63985, upload-time = "2026-05-08T21:01:53.847Z" }, + { url = "https://files.pythonhosted.org/packages/c8/54/c317819ec157cbf6f35df9df9657a6f82daf34d5faf15948b2f639c2192e/propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285", size = 63999, upload-time = "2026-05-08T21:01:55.179Z" }, + { url = "https://files.pythonhosted.org/packages/5a/56/387e3f7dfce0a9233df41fb888aa1c30222cb4bbbf09537c02dd9bd85fe2/propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837", size = 62779, upload-time = "2026-05-08T21:01:57.489Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9c/596784cb5824ed61ee960d3f8655a3f0993e107c6e98ab6c818b7fb92ccb/propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8", size = 59796, upload-time = "2026-05-08T21:01:58.736Z" }, + { url = "https://files.pythonhosted.org/packages/c2/3d/1a6cfa1726a48542c1e8784a0761421476a5b68e09b7f36bf95eb954aaba/propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366", size = 66023, upload-time = "2026-05-08T21:02:00.228Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0e/05fd6990369477076e4e280bcb970de760fddf0161a46e988bc95f7940ec/propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56", size = 64448, upload-time = "2026-05-08T21:02:01.888Z" }, + { url = "https://files.pythonhosted.org/packages/cd/86/5f8da315a4309c62c10c0b2516b17492d5d3bbe1bb862b96604db67e2a37/propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d", size = 67329, upload-time = "2026-05-08T21:02:03.484Z" }, + { url = "https://files.pythonhosted.org/packages/da/d3/3368efe79ab21f0cdf86ef49895811c9cc933131d4cde1f28a624e22e712/propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2", size = 65172, upload-time = "2026-05-08T21:02:04.745Z" }, + { url = "https://files.pythonhosted.org/packages/d5/07/127e8b0bacfb325396196f9d976a22453049b89b9b2b08477cc3145faa44/propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821", size = 43813, upload-time = "2026-05-08T21:02:06.025Z" }, + { url = "https://files.pythonhosted.org/packages/88/fb/46dad6c0ae49ed230ab1b16c890c2b6314e2403e6c412976f4a72d64a527/propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370", size = 47764, upload-time = "2026-05-08T21:02:07.353Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c4/a47d0a63aa309d10d59ede6e9d4cff03a344a79d1f0f4cd0cd74997b53e0/propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6", size = 41140, upload-time = "2026-05-08T21:02:09.065Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, +] + +[[package]] +name = "proto-plus" +version = "1.28.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/3e/29e0d6a2c5adde6ab5772253fd16ab346324026b89a66e354689c86d0584/proto_plus-1.28.2.tar.gz", hash = "sha256:26d843eb99c1e32fdf1d20ff0faae56607f7748fe774acf9ecd5cfe6c6472501", size = 58063, upload-time = "2026-07-22T16:28:29.119Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/84/4e9a53a062d4073c74897a6bd20fff74d55307341b3e85c081002462b3ef/proto_plus-1.28.2-py3-none-any.whl", hash = "sha256:b874236fcac2358f601e4330bcb76cb8b89c851303ccf4078408b3d4774d1c52", size = 50693, upload-time = "2026-07-22T16:28:24.059Z" }, +] + +[[package]] +name = "protobuf" +version = "7.35.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/01/9ef0afd7999eb9badb3a768b4aedd78c86d4c65cfaf1958ab276199e76b4/protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a", size = 458717, upload-time = "2026-06-11T21:55:40.257Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/03/8aeeb7458d22546bf64b5250ca1daeb5ff757d900e8e4a7476c6f0db843e/protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6", size = 433226, upload-time = "2026-06-11T21:55:31.719Z" }, + { url = "https://files.pythonhosted.org/packages/37/4b/dfb89eb0e652a1ff073c39a59fb5e3a83cfe9b57a2c83fa6d78270101767/protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799", size = 328847, upload-time = "2026-06-11T21:55:34.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/58/dc12f2cd484951524af6e3382c785869b9b3fb5e52ee95ae23add53ee8f9/protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4", size = 344030, upload-time = "2026-06-11T21:55:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/e4/be/5b3cfe508bfab6761414ff944e3366eb13be4fd71efcd69450f89ba39f43/protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4", size = 327130, upload-time = "2026-06-11T21:55:35.921Z" }, + { url = "https://files.pythonhosted.org/packages/d8/bc/6d6c7ba8709c85f8f2c390b2b118d6fb08a783676a572271851bf45a7d22/protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30", size = 428945, upload-time = "2026-06-11T21:55:37.034Z" }, + { url = "https://files.pythonhosted.org/packages/0a/19/8d0cb6f20a1ef7b18f1c8986ad5783f22f84cce39c6ce9a6e645ea55192e/protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87", size = 439996, upload-time = "2026-06-11T21:55:38.123Z" }, + { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" }, +] + +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + +[[package]] +name = "ptyprocess" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762, upload-time = "2020-12-28T15:15:30.155Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" }, +] + +[[package]] +name = "pure-eval" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/05/0a34433a064256a578f1783a10da6df098ceaa4a57bbeaa96a6c0352786b/pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42", size = 19752, upload-time = "2024-07-21T12:58:21.801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, +] + +[[package]] +name = "pyarrow" +version = "25.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/f3/95428098d1fa7d04432fb750eed06b41304c2f6a5d3319985e64db2d9d41/pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4", size = 1199181, upload-time = "2026-07-10T08:29:50.116Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/44/fdd3a4377807b7dcabe2d4b5aa99dbbc98e2e5df3f1ca4e7f0aec492d987/pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e", size = 35850884, upload-time = "2026-07-10T08:26:47.357Z" }, + { url = "https://files.pythonhosted.org/packages/bf/71/9f053177a7709b8c90abb00a2375b916286f9f0d6cfb21a5cadd4ef811e8/pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec", size = 37616197, upload-time = "2026-07-10T08:26:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/95/1a/22bfb6597dcdc861fa83c39c06e1457cb56f698940eff42fbb25de30e8e5/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f", size = 46841966, upload-time = "2026-07-10T08:27:07.685Z" }, + { url = "https://files.pythonhosted.org/packages/55/0e/cd705c042bc4fe7022478db577fcab4abdcfabb9bc37ab7a75556b3fcb2b/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778", size = 50088993, upload-time = "2026-07-10T08:27:14.268Z" }, + { url = "https://files.pythonhosted.org/packages/98/ee/d822e1ee31fe31ec5d057210e0605c950b975dcd8d9a332976cc859a9df8/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537", size = 49941005, upload-time = "2026-07-10T08:27:21.274Z" }, + { url = "https://files.pythonhosted.org/packages/33/1b/207a90cc64619a095eb75a263ae069735f2810056d43c667befd573ec083/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b", size = 53112355, upload-time = "2026-07-10T08:27:27.911Z" }, + { url = "https://files.pythonhosted.org/packages/7e/fe/81d1e5f8beed15c01e98649d5c6e2167b67fd395884a2488f18bf1cf0dba/pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5", size = 27945954, upload-time = "2026-07-10T08:27:32.903Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c8/098ce17d778fd9d29e40bb8c5f19a40cc90c3f0b46c9057b0d7993f42f54/pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc", size = 35844549, upload-time = "2026-07-10T08:27:37.956Z" }, + { url = "https://files.pythonhosted.org/packages/bc/66/24c28877219abf6263d909b1592c97ff82c59f13a59acbed11fc87c0654f/pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e", size = 37610397, upload-time = "2026-07-10T08:27:43.803Z" }, + { url = "https://files.pythonhosted.org/packages/53/55/6d1d5f5aff317ec5de9421594679ed51ed828fe7e2ce209327f819d801e4/pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be", size = 46841701, upload-time = "2026-07-10T08:27:49.741Z" }, + { url = "https://files.pythonhosted.org/packages/b5/5d/f790fb6965ab54c9da0dda7856abc75fd0d7648d865f8d603c111d203a64/pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9", size = 50090118, upload-time = "2026-07-10T08:27:56.051Z" }, + { url = "https://files.pythonhosted.org/packages/0c/8c/faf025357ebf31bc96777f234277aa31e2aeca6dd4ecaa391f29085473c2/pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d", size = 49945559, upload-time = "2026-07-10T08:28:01.927Z" }, + { url = "https://files.pythonhosted.org/packages/07/a1/bd051871708ea99a5e0fc711926c26c6f2c6d0130c7aaac8093e34998af6/pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1", size = 53114238, upload-time = "2026-07-10T08:28:08.594Z" }, + { url = "https://files.pythonhosted.org/packages/7c/31/737f0c3cffcd6af647849477d1dd68045deac2e3963c3f9f211bedc48540/pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18", size = 27861162, upload-time = "2026-07-10T08:28:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/55/c7/581ccbcdb3d897eb2893328d68db3d52eca373bf2a7e964d0a6276b8e85b/pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b", size = 35878945, upload-time = "2026-07-10T08:28:18.222Z" }, + { url = "https://files.pythonhosted.org/packages/64/d1/ccb01db7329ea0411ef4fbd9b62a04d3268b36777d4e758d5e39b91ddeab/pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b", size = 37630854, upload-time = "2026-07-10T08:28:23.452Z" }, + { url = "https://files.pythonhosted.org/packages/af/9f/2d81ba89d1e4198d0cb25fe7529de936830fdaec0db926bb52a1ef7080d4/pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec", size = 46905617, upload-time = "2026-07-10T08:28:29.376Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/0ed312ec800fb536f93783215126cee4b8977dcfeccba6f0f44df0cc87d7/pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887", size = 50119765, upload-time = "2026-07-10T08:28:35.826Z" }, + { url = "https://files.pythonhosted.org/packages/ca/88/cab5063ba0c4d46a9f6b4b7eb1c9029dc0302d65cd5ab3510c949a386568/pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62", size = 50027563, upload-time = "2026-07-10T08:28:43.848Z" }, + { url = "https://files.pythonhosted.org/packages/7b/fb/4d24f1b7fe2e042dc4ef315ef75e4e702d8e46fe10c37e63caff00502b03/pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f", size = 53162437, upload-time = "2026-07-10T08:28:52.819Z" }, + { url = "https://files.pythonhosted.org/packages/fa/65/da20806de93ca6ee91e72cb6a9b08b3ac890b46efc8d94a7326c651c4c81/pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104", size = 28613262, upload-time = "2026-07-10T08:29:47.544Z" }, + { url = "https://files.pythonhosted.org/packages/86/9f/c632afb1d3ef4a7814cee236718235f3a47eac46e97eb87df40f550b6b48/pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e", size = 36120702, upload-time = "2026-07-10T08:28:59.577Z" }, + { url = "https://files.pythonhosted.org/packages/36/0a/093d53a0e72ad06e45d6443e00651bbc2d21af4211295086cbf4d873d3b9/pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2", size = 37750674, upload-time = "2026-07-10T08:29:06.916Z" }, + { url = "https://files.pythonhosted.org/packages/8a/18/b37fc31a69cff4bdfb8842683def5612f551b93fff6f44375e4a4a6a5535/pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50", size = 46912304, upload-time = "2026-07-10T08:29:14.656Z" }, + { url = "https://files.pythonhosted.org/packages/32/35/5cae19ba72493e5598022468b56f6a5571f399f485bf412f157356476caa/pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe", size = 50073652, upload-time = "2026-07-10T08:29:22.467Z" }, + { url = "https://files.pythonhosted.org/packages/2e/a5/ddd508424bdfd5e6945765e9e2ffc687e2f6115972badc8ecf423076c407/pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc", size = 50058654, upload-time = "2026-07-10T08:29:29.689Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a4/324d0db203ff5eebe8694ec2d6ec5a23f9aaa5d02e5b8c692914c518c33c/pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0", size = 53140153, upload-time = "2026-07-10T08:29:36.041Z" }, + { url = "https://files.pythonhosted.org/packages/bd/8d/d236e9c82fe315f9128885c8be3ec719f41965a1eb6b6f4b42470904cd41/pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849", size = 28743657, upload-time = "2026-07-10T08:29:42.742Z" }, +] + +[[package]] +name = "pyasn1" +version = "0.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + +[[package]] +name = "pycocotools" +version = "2.0.11" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a2/df/32354b5dda963ffdfc8f75c9acf8828ef7890723a4ed57bb3ff2dc1d6f7e/pycocotools-2.0.11.tar.gz", hash = "sha256:34254d76da85576fcaf5c1f3aa9aae16b8cb15418334ba4283b800796bd1993d", size = 25381, upload-time = "2025-12-15T22:31:46.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/12/2f2292332456e4e4aba1dec0e3de8f1fc40fb2f4fdb0ca1cb17db9861682/pycocotools-2.0.11-cp312-abi3-macosx_10_13_universal2.whl", hash = "sha256:a2e9634bc7cadfb01c88e0b98589aaf0bd12983c7927bde93f19c0103e5441f4", size = 147795, upload-time = "2025-12-15T22:31:11.519Z" }, + { url = "https://files.pythonhosted.org/packages/63/3c/68d7ea376aada9046e7ea2d7d0dad0d27e1ae8b4b3c26a28346689390ab2/pycocotools-2.0.11-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7fd4121766cc057133534679c0ec3f9023dbd96e9b31cf95c86a069ebdac2b65", size = 398434, upload-time = "2025-12-15T22:31:12.558Z" }, + { url = "https://files.pythonhosted.org/packages/23/59/dc81895beff4e1207a829d40d442ea87cefaac9f6499151965f05c479619/pycocotools-2.0.11-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a82d1c9ed83f75da0b3f244f2a3cf559351a283307bd9b79a4ee2b93ab3231dd", size = 411685, upload-time = "2025-12-15T22:31:13.995Z" }, + { url = "https://files.pythonhosted.org/packages/0b/0b/5a8a7de300862a2eb5e2ecd3cb015126231379206cd3ebba8f025388d770/pycocotools-2.0.11-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:89e853425018e2c2920ee0f2112cf7c140a1dcf5f4f49abd9c2da112c3e0f4b3", size = 390500, upload-time = "2025-12-15T22:31:15.138Z" }, + { url = "https://files.pythonhosted.org/packages/63/b5/519bb68647f06feea03d5f355c33c05800aeae4e57b9482b2859eb00752e/pycocotools-2.0.11-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:87af87b8d06d5b852a885a319d9362dca3bed9f8bbcc3feb6513acb1f88ea242", size = 409790, upload-time = "2025-12-15T22:31:16.326Z" }, + { url = "https://files.pythonhosted.org/packages/83/b4/f6708404ff494706b80e714b919f76dc4ec9845a4007affd6d6b0843f928/pycocotools-2.0.11-cp312-abi3-win_amd64.whl", hash = "sha256:ffe806ce535f5996445188f9a35643791dc54beabc61bd81e2b03367356d604f", size = 77570, upload-time = "2025-12-15T22:31:17.703Z" }, + { url = "https://files.pythonhosted.org/packages/6e/63/778cd0ddc9d4a78915ac0a72b56d7fb204f7c3fabdad067d67ea0089762e/pycocotools-2.0.11-cp312-abi3-win_arm64.whl", hash = "sha256:c230f5e7b14bd19085217b4f40bba81bf14a182b150b8e9fab1c15d504ade343", size = 64564, upload-time = "2025-12-15T22:31:18.652Z" }, + { url = "https://files.pythonhosted.org/packages/5d/78/31c81e99d596a20c137d8a2e7a25f39a88f88fada5e0b253fce7323ecf0d/pycocotools-2.0.11-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:fd72b9734e6084b217c1fc3945bfd4ec05bdc75a44e4f0c461a91442bb804973", size = 168931, upload-time = "2025-12-15T22:31:19.845Z" }, + { url = "https://files.pythonhosted.org/packages/5f/63/fdd488e4cd0fdc6f93134f2cd68b1fce441d41566e86236bf6156961ef9b/pycocotools-2.0.11-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7eb43b79448476b094240450420b7425d06e297880144b8ea6f01e9b4340e43", size = 484856, upload-time = "2025-12-15T22:31:21.231Z" }, + { url = "https://files.pythonhosted.org/packages/a1/fc/c83648a8fb7ea3b8e2ce2e761b469807e6cadb81577bf1af31c4f2ef0d87/pycocotools-2.0.11-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c3546b93b39943347c4f5b0694b5824105cbe2174098a416bcad4acd9c21e957", size = 480994, upload-time = "2025-12-15T22:31:22.426Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2d/35e1122c0d007288aa9545be9549cbc7a4987b2c22f21d75045260a8b5b8/pycocotools-2.0.11-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:efd1694b2075f2f10c5828f10f6e6c4e44368841fd07dae385c3aa015c8e25f9", size = 467956, upload-time = "2025-12-15T22:31:23.754Z" }, + { url = "https://files.pythonhosted.org/packages/e4/ff/30cfe8142470da3e45abe43a9842449ca0180d993320559890e2be19e4a5/pycocotools-2.0.11-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:368244f30eb8d6cae7003aa2c0831fbdf0153664a32859ec7fbceea52bfb6878", size = 474658, upload-time = "2025-12-15T22:31:24.883Z" }, + { url = "https://files.pythonhosted.org/packages/bc/62/254ca92604106c7a5af3258e589e465e681fe0166f9b10f97d8ca70934d6/pycocotools-2.0.11-cp313-cp313t-win_amd64.whl", hash = "sha256:ac8aa17263e6489aa521f9fa91e959dfe0ea3a5519fde2cbf547312cdce7559e", size = 89681, upload-time = "2025-12-15T22:31:26.025Z" }, + { url = "https://files.pythonhosted.org/packages/6e/f0/c019314dc122ad5e6281de420adc105abe9b59d00008f72ef3ad32b1e328/pycocotools-2.0.11-cp313-cp313t-win_arm64.whl", hash = "sha256:04480330df5013f6edd94891a0ee8294274185f1b5093d1b0f23d51778f0c0e9", size = 70520, upload-time = "2025-12-15T22:31:26.999Z" }, + { url = "https://files.pythonhosted.org/packages/66/2b/58b35c88f2086c043ff1c87bd8e7bf36f94e84f7b01a5e00b6f5fabb92a7/pycocotools-2.0.11-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:a6b13baf6bfcf881b6d6ac6e23c776f87a68304cd86e53d1d6b9afa31e363c4e", size = 169883, upload-time = "2025-12-15T22:31:28.233Z" }, + { url = "https://files.pythonhosted.org/packages/24/c0/b970eefb78746c8b4f8b3fa1b49d9f3ec4c5429ef3c5d4bbcc55abebe478/pycocotools-2.0.11-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78bae4a9de9d34c4759754a848dfb3306f9ef1c2fcb12164ffbd3d013d008321", size = 486894, upload-time = "2025-12-15T22:31:29.283Z" }, + { url = "https://files.pythonhosted.org/packages/5b/f7/db7436820a1948d96fa9764b6026103e808840979be01246049f2c1e7f94/pycocotools-2.0.11-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83d896f4310379849dfcfa7893afb0ff21f4f3cdb04ab3f61b05dd98953dd0ad", size = 483249, upload-time = "2025-12-15T22:31:31.687Z" }, + { url = "https://files.pythonhosted.org/packages/1e/a6/a14a12c9f50c41998fdc0d31fd3755bcbce124bac9abb1d6b99d1853cafd/pycocotools-2.0.11-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:eebd723503a2eb2c8b285f56ea3be1d9f3875cd7c40d945358a428db94f14015", size = 469070, upload-time = "2025-12-15T22:31:32.821Z" }, + { url = "https://files.pythonhosted.org/packages/46/de/aa4f65ece3da8e89310a1be00cad0700170fd13f41a3aaae2712291269d5/pycocotools-2.0.11-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:bd7a1e19ef56a828a94bace673372071d334a9232cd32ae3cd48845a04d45c4f", size = 475589, upload-time = "2025-12-15T22:31:34.188Z" }, + { url = "https://files.pythonhosted.org/packages/44/6f/04a30df03ae6236b369b361df0c50531d173d03678978806aa2182e02d1e/pycocotools-2.0.11-cp314-cp314t-win_amd64.whl", hash = "sha256:63026e11a56211058d0e84e8263f74cbccd5e786fac18d83fd221ecb9819fcc7", size = 93863, upload-time = "2025-12-15T22:31:35.38Z" }, + { url = "https://files.pythonhosted.org/packages/da/05/8942b640d6307a21c3ede188e8c56f07bedf246fac0e501437dbda72a350/pycocotools-2.0.11-cp314-cp314t-win_arm64.whl", hash = "sha256:8cedb8ccb97ffe9ed2c8c259234fa69f4f1e8665afe3a02caf93f6ef2952c07f", size = 72038, upload-time = "2025-12-15T22:31:36.768Z" }, +] + +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, +] + +[[package]] +name = "pyee" +version = "13.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pylatexenc" +version = "3.0b2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/51/68a89d104d9b892bf459912367256b1c5222aca544f655303a1981d94755/pylatexenc-3.0b2.tar.gz", hash = "sha256:14c6148c3cabebaff5e488e304abb21024f86fdab7d70bafc9f29d62463a7d9d", size = 260045, upload-time = "2026-07-28T15:26:14.179Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/f9/44351165a7f14073e75825769fd1aec61d1da944b61a7296725cc95ff889/pylatexenc-3.0b2-py3-none-any.whl", hash = "sha256:0cd14a2104ec4eaeaaadcab85b2da8c183a5feb39d71301a4db9d033b3e6e349", size = 322323, upload-time = "2026-07-28T15:26:13.172Z" }, +] + +[[package]] +name = "pymupdf" +version = "1.28.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/e9/6d6c5d6c0a3551bffd47681a6240caf941727f195b45593cf20ab36f018f/pymupdf-1.28.0.tar.gz", hash = "sha256:e53f3567403a92da15caa9e7ae0164327fff48817e9f40175367fb9de524258d", size = 87637751, upload-time = "2026-06-29T09:08:47.547Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/b7/88043e38cc7529de070f0c9bd267fa258035cca0b4ad5260536b994594a7/pymupdf-1.28.0-cp310-abi3-macosx_10_15_x86_64.whl", hash = "sha256:892b89ba88e8f98b53133b62877a9dc9b5e7dc6a4aeb837b612db56a8d2e03ac", size = 24597385, upload-time = "2026-06-29T09:03:30.608Z" }, + { url = "https://files.pythonhosted.org/packages/33/f4/23775bbda0781b61fc398cc75079a2b0e64696d8fcf93271748883e9627e/pymupdf-1.28.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:4d692dcf44d3566ae96bc6f6346c6ad432274a29ba617bf7a9fe18009e24adb4", size = 23828292, upload-time = "2026-06-29T09:03:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/1c/f5/bf75fc7a415722f8b33662054f82d88520c0cbfd4c36d0e08aeaec605e49/pymupdf-1.28.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:47a5c29ed4eb0744de9c4e37bb49b1259b18d4d75fcc8a7c130f7c9fa15956f6", size = 25045507, upload-time = "2026-06-29T09:04:03.86Z" }, + { url = "https://files.pythonhosted.org/packages/58/69/5d12c9f1f2d76f28383d6110a069c79fbfced5a4f97bb1ee6e8354f52bb7/pymupdf-1.28.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:44f0973f5e5edbaec95bc34b64e71d1959d4ee90b1328de1b4f4f5b4fa78673f", size = 25716599, upload-time = "2026-06-29T09:04:19.367Z" }, + { url = "https://files.pythonhosted.org/packages/4d/b4/ec0e017bc42857cc86bd651441dbc41cc18be48d4698ecd27aac491e0c9a/pymupdf-1.28.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4d61ec323a706e153a12e262e51febfb43eeaa20977785ace135d18d48bcdc83", size = 25940489, upload-time = "2026-06-29T09:04:36.624Z" }, + { url = "https://files.pythonhosted.org/packages/06/86/f831fef09013f33b3c9c09fb3923f2ff53e1e437f6ace14b8ae46392f558/pymupdf-1.28.0-cp310-abi3-win32.whl", hash = "sha256:caea2b3b67347fd79e5d15ed7929b0e886aac594ea228073b6d39de0078189da", size = 18489703, upload-time = "2026-06-29T20:50:30.599Z" }, + { url = "https://files.pythonhosted.org/packages/2e/5d/1a03f53eb0449900469335fcfc742ca28e3ba159b7d650e0921d50b8b308/pymupdf-1.28.0-cp310-abi3-win_amd64.whl", hash = "sha256:e01e90fd86abfeb37ceb921eddb951f988a11d45ff6ce6b7664f2039849068ec", size = 19773102, upload-time = "2026-06-29T09:04:49.773Z" }, + { url = "https://files.pythonhosted.org/packages/72/f6/1e52ce243ca792254f6223b4017c5667194c146ce9b88baf37bc5eb3d1c9/pymupdf-1.28.0-cp313-abi3-pyemscripten_2025_0_wasm32.whl", hash = "sha256:74c6d00ba2a9aad3a635db73b07c15db462b480741d831a34a75a56535ebc22b", size = 18357011, upload-time = "2026-06-29T20:50:50.353Z" }, + { url = "https://files.pythonhosted.org/packages/62/b1/46b5b3d8ef3cc71114667cf10c4d8b33f39af97253af32e9a0986775b638/pymupdf-1.28.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:b3e1399c7a64c6914239116a369efcdaac4cfb9e838bde2656d7accc4a85c72d", size = 25753599, upload-time = "2026-06-29T09:05:09.398Z" }, +] + +[[package]] +name = "pyparsing" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, +] + +[[package]] +name = "pypdf" +version = "6.14.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/72/7dfd5ff1c9c37de97a731701f51af091325f123d9d4270361c9c69e4431f/pypdf-6.14.2.tar.gz", hash = "sha256:7873f502fe4385e79539b21d872392dc0c4e3714327c15881cbc7fbfd1f95b25", size = 6491182, upload-time = "2026-06-23T14:18:30.859Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/e6/136aa8993a2ae7214e0b0ef2edaa0d2e08d1d4e4982635b08a835ff31ec8/pypdf-6.14.2-py3-none-any.whl", hash = "sha256:3f07891af76dc002657e04993ab9b4de81de29f9013b9761d0b7968bff12e946", size = 349514, upload-time = "2026-06-23T14:18:28.867Z" }, +] + +[[package]] +name = "pypdfium2" +version = "5.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/42/0b51bdf50ccf13f3deb3209ca996179a49761dc191748469cf0de55b0055/pypdfium2-5.12.1.tar.gz", hash = "sha256:d0e0648fb2e28f50efcd1ec0a5a18ced9f4d66b2c227fae9b603f0a883b2d13f", size = 274428, upload-time = "2026-07-17T10:01:22.713Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/7e/bd8df53b1131c582f6646372047b49162032bd01628d49b9a60cd94d2181/pypdfium2-5.12.1-py3-none-android_23_arm64_v8a.whl", hash = "sha256:05bab9b1ba2de7fc299ae2af25cb9c8a0543bc8bb893e879fe8c9ba8310e9ce4", size = 3392276, upload-time = "2026-07-17T10:00:47.376Z" }, + { url = "https://files.pythonhosted.org/packages/b2/13/a2b71e17b0439d2af78c817a381e3557371c6c56098581da8485aef65ea6/pypdfium2-5.12.1-py3-none-android_23_armeabi_v7a.whl", hash = "sha256:d4ee061e566a6422b660cdddaaa799a2d1cbf2f016921bcaf24d61426d01d942", size = 2848776, upload-time = "2026-07-17T10:00:49.09Z" }, + { url = "https://files.pythonhosted.org/packages/e2/a8/d7a61700db3022792b28bce5264be90a7d2e7104998362af6b93766f50b2/pypdfium2-5.12.1-py3-none-macosx_13_0_arm64.whl", hash = "sha256:66a9ed40d70a5d728cd42148fecb9d7a0917c6161d6bb67c844093a4ed1df089", size = 3480243, upload-time = "2026-07-17T10:00:50.674Z" }, + { url = "https://files.pythonhosted.org/packages/01/2c/d7a38fad74b6da0947cf8763aee0f8e6c9d3c12fc8e137aa615f7f8ae76c/pypdfium2-5.12.1-py3-none-macosx_13_0_x86_64.whl", hash = "sha256:847378a5ab41332998b2621b21bab2e96dc8c3eff36a08bce26695b964163983", size = 3643490, upload-time = "2026-07-17T10:00:52.236Z" }, + { url = "https://files.pythonhosted.org/packages/8f/29/aca739676323558595fcf8cdc8d7939d2b25aaaa6f538e829f1fee938cdd/pypdfium2-5.12.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6eabf028ad8e7bc7811c9acf3a72718c180569b624b844d2c6cc974609784275", size = 3649734, upload-time = "2026-07-17T10:00:53.776Z" }, + { url = "https://files.pythonhosted.org/packages/a4/e0/e4ecc05f4f1a11d11c8d684a24b2fc8be8207f341be985dac301caa4f6aa/pypdfium2-5.12.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7857cfa6642ec5a09db12ff8f5cf6b6494585b5e3a605399fddc4fb862837b63", size = 3380828, upload-time = "2026-07-17T10:00:55.377Z" }, + { url = "https://files.pythonhosted.org/packages/17/1b/c94c9d486791276e736350917a11fe2cf3acba2c6c7f03f9ac0d51f8952a/pypdfium2-5.12.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05bfa20a08a96584253bbe38b60e13f81a037eac31c5579e607ec1480ad25dbf", size = 3777202, upload-time = "2026-07-17T10:00:57.212Z" }, + { url = "https://files.pythonhosted.org/packages/14/aa/7f81f0c035fc32850dfab9daf78530814f215be226dad7491e3caa0a3e8c/pypdfium2-5.12.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9f059f7bdbdf4352eb83691071096940d769d6ae5930b8734237fdb1bd78fbc2", size = 4186083, upload-time = "2026-07-17T10:00:59.022Z" }, + { url = "https://files.pythonhosted.org/packages/23/16/21420a6f2bc5f981299c336817dd5d72709dad5fda30ac38cbd5f0f7b372/pypdfium2-5.12.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e10cbf41b21233ec5e20adfc170cf60edd77abead86a97dc708fff55a8a886c7", size = 3701734, upload-time = "2026-07-17T10:01:00.952Z" }, + { url = "https://files.pythonhosted.org/packages/36/a1/bb89f49e2b3ea3e945b67859d2ec6e73e722a83c006099c675692641e51d/pypdfium2-5.12.1-py3-none-manylinux_2_27_s390x.manylinux_2_28_s390x.whl", hash = "sha256:07eeebb2784f4cd38d386b924235df43217a397442796673296bb6efbdaad1d0", size = 4030403, upload-time = "2026-07-17T10:01:02.568Z" }, + { url = "https://files.pythonhosted.org/packages/80/a7/cea5eb0c39e9c6fdf9853a3008bf021d08f962228073af90354b60c5ccdc/pypdfium2-5.12.1-py3-none-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5c3e6cbe43581af79526184643920ab03a9401a0c79f2226bea9d4d1e3d34008", size = 3994411, upload-time = "2026-07-17T10:01:04.25Z" }, + { url = "https://files.pythonhosted.org/packages/fc/43/5e470213b27c13b0d94d03d97fc3740507edadea1d1e2ce2049bfbec4aa0/pypdfium2-5.12.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4648f0905441bcb141687ca2263bbf38a1aa056b943eef06019f91cff3e1da4a", size = 4993687, upload-time = "2026-07-17T10:01:05.811Z" }, + { url = "https://files.pythonhosted.org/packages/db/da/af7972ca72f24ed720db6501333fd67bc66aa6aa5a5ec698551bd30ae62e/pypdfium2-5.12.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:bdff622181fab64f32328591c9c8287cdc745c9a1f2afc26ca3feba39e3e6645", size = 4534560, upload-time = "2026-07-17T10:01:07.291Z" }, + { url = "https://files.pythonhosted.org/packages/a9/63/06a7f2cd691f7e336cbc53fd65453fee516042c8ddb016bc50d1cd2bed45/pypdfium2-5.12.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:236dbdc88aa54f14b27937ccb2ebe3dcf08c10dbb8652f432ea982dc9af39732", size = 5237681, upload-time = "2026-07-17T10:01:08.997Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f5/937b080671758ab0b3d3d69b2657006682e4c6a0134b36774be4ed1afcfb/pypdfium2-5.12.1-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:9c8856ce7dd77a7827476c7d75afe1197d6cd505f5cb4167b6aacf661f3f8ea5", size = 5143027, upload-time = "2026-07-17T10:01:10.69Z" }, + { url = "https://files.pythonhosted.org/packages/32/02/094632700c24728fa443dc89d9d1c6e4fc05bb00778b22e8482fdb133da0/pypdfium2-5.12.1-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:5f257bb40fa44ce9ba18d2c919777dbd3f16bf22548b1d68fd56c7c92f1de530", size = 4647048, upload-time = "2026-07-17T10:01:12.559Z" }, + { url = "https://files.pythonhosted.org/packages/fe/d0/12d84bf55a4fcf2c0ed242afc94168933194f672067e1d162aa60a8e4426/pypdfium2-5.12.1-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:974082344172da76a5c3c0782eaedfe6069dbe88db77d8c671ef36b61e9b14e2", size = 5088747, upload-time = "2026-07-17T10:01:14.42Z" }, + { url = "https://files.pythonhosted.org/packages/92/9c/92a460bac1f6cfd6f96251802b3098a804fb251dfe0b5eb004ede958ae0e/pypdfium2-5.12.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:715ae16b34ea1d64884d58800155179ba700e9ea65a2f583b020666acd2bfb12", size = 5049695, upload-time = "2026-07-17T10:01:15.961Z" }, + { url = "https://files.pythonhosted.org/packages/f7/67/53c61d366222550220b42a9212131407f49d3dbcf050178c620bf80fa899/pypdfium2-5.12.1-py3-none-win32.whl", hash = "sha256:e5358d2ce4ebc5c899aab1df9ca5d215357244e9168aa443225d3c1e649c7eac", size = 3725466, upload-time = "2026-07-17T10:01:17.773Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c3/08b62718faf2f6b6aa49207626e3113ff3ec1b3cd076c0ef8fd852f0e57c/pypdfium2-5.12.1-py3-none-win_amd64.whl", hash = "sha256:9609be73a6701a68f29dffe0335f7a2e4b3ba581542ed65d35d49f761a4600ca", size = 3859845, upload-time = "2026-07-17T10:01:19.417Z" }, + { url = "https://files.pythonhosted.org/packages/6c/d5/ad551e55790134c8bc87ea16e0866a3721def0620fbfa19112c8ad4a25a6/pypdfium2-5.12.1-py3-none-win_arm64.whl", hash = "sha256:afc0b7e0c975a429abc75875209ce17b66d749f6ac5cbe8ba72470e83901e304", size = 3674605, upload-time = "2026-07-17T10:01:21.008Z" }, +] + +[[package]] +name = "pysocks" +version = "1.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/11/293dd436aea955d45fc4e8a35b6ae7270f5b8e00b53cf6c024c83b657a11/PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0", size = 284429, upload-time = "2019-09-20T02:07:35.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/59/b4572118e098ac8e46e399a1dd0f2d85403ce8bbaad9ec79373ed6badaf9/PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", size = 16725, upload-time = "2019-09-20T02:06:22.938Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "rapidfuzz" +version = "3.14.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/21/ef6157213316e85790041254259907eb722e00b03480256c0545d98acd33/rapidfuzz-3.14.5.tar.gz", hash = "sha256:ba10ac57884ce82112f7ed910b67e7fb6072d8ef2c06e30dc63c0f604a112e0e", size = 57901753, upload-time = "2026-04-07T11:16:31.931Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/e3/574435c6aafb80254c191ef40d7aca2cb2bb97a095ec9395e9fa59ac307a/rapidfuzz-3.14.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0d3378f471ef440473a396ce2f8e97ee12f89a78b495540e0a5617bbfe895638", size = 1944601, upload-time = "2026-04-07T11:14:18.771Z" }, + { url = "https://files.pythonhosted.org/packages/d0/1f/fbad3102a255ecc112ce9a7e779bacab7fd14398217be8868dc9082ba363/rapidfuzz-3.14.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e910eebca9fd0eba245c0555e764597e8a0cccb673a92da2dc2397050725f48", size = 1164293, upload-time = "2026-04-07T11:14:20.534Z" }, + { url = "https://files.pythonhosted.org/packages/88/37/a3eb7ff6121ed3a5f199a8c38cc86c8e481816f879cb0e0b738b078c9a7e/rapidfuzz-3.14.5-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:01550fe5f60fd176aa66b7611289d46dc4aa4b1b904874c7b6d1d54e581c5ec1", size = 1371999, upload-time = "2026-04-07T11:14:22.63Z" }, + { url = "https://files.pythonhosted.org/packages/79/72/97a9728c711c7c1b06e107d3f0623880fb4ef90e147ed13c551a1730e7cc/rapidfuzz-3.14.5-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48bee0b91bebfaec41e1081e351000659ab7570cc4598d617aa04d5bf827f9e6", size = 3145715, upload-time = "2026-04-07T11:14:24.508Z" }, + { url = "https://files.pythonhosted.org/packages/ed/54/d5caabbea233ac90c286c87c260e49d7641467e87438a18d858e41c82e91/rapidfuzz-3.14.5-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:7e580cb04ad849ae9b786fa21383c6b994b6e6c1444ad1cb9f22392759d72741", size = 1456304, upload-time = "2026-04-07T11:14:26.515Z" }, + { url = "https://files.pythonhosted.org/packages/fc/a7/2d1a81250ac8c01a0100c026018e76f0e7a097ff63e4c553e02a6938c6fb/rapidfuzz-3.14.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:09d6c9ba091854f07817055d795d604179c12a8f308ba4c7d56f3719dfea1646", size = 2389089, upload-time = "2026-04-07T11:14:28.635Z" }, + { url = "https://files.pythonhosted.org/packages/65/0d/c47c3872203ae88e6506997c0b576ad731f5261daa25d559be09c9756658/rapidfuzz-3.14.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:1e989f86113be66574113b9c7bdf4793f3f863d248e47d911b355e05ca6b6b10", size = 2493404, upload-time = "2026-04-07T11:14:30.577Z" }, + { url = "https://files.pythonhosted.org/packages/8f/2f/71e0a5a3130792146c8a200a2dd1e52aa16f7c1074012e17f2601eea9a90/rapidfuzz-3.14.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ebd1a18e2e47bc0b292a07e6ed9c3642f8aaa672d12253885f599b50807a4f9", size = 4251709, upload-time = "2026-04-07T11:14:32.451Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/d39874901abacef325adb5b34ae416817c8486dfb4fb87c7a9b74ec5b072/rapidfuzz-3.14.5-cp312-cp312-win32.whl", hash = "sha256:9981d38a703b86f0e315a3cd229fd1906fe1d91c989ed121fb975b3c849f89f5", size = 1710069, upload-time = "2026-04-07T11:14:34.37Z" }, + { url = "https://files.pythonhosted.org/packages/85/0b/f65572c53de8a1c704bda707f63a447b67bdbe95d7cdc70d18885e191df5/rapidfuzz-3.14.5-cp312-cp312-win_amd64.whl", hash = "sha256:d8375e3da319593389727c3187ccaf3e0e84199accc530866b8e0f2b79af05e9", size = 1540630, upload-time = "2026-04-07T11:14:36.287Z" }, + { url = "https://files.pythonhosted.org/packages/5e/c3/143be3a578f989758cae516f3270d5cbb49783a7bfdf57cc27a670e00456/rapidfuzz-3.14.5-cp312-cp312-win_arm64.whl", hash = "sha256:478b59bb018a6780d73f33e38d0b3ec5e968a6c1ed42876b993dd456b7aa20e8", size = 813137, upload-time = "2026-04-07T11:14:38.289Z" }, + { url = "https://files.pythonhosted.org/packages/11/66/252803f2010ba699618cdc048b6e1f7cc1f433c08b4a9a17579b92ab0142/rapidfuzz-3.14.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ebd8fd343bf8492a1e60bcb6dc99f90f74f65d98d8241a6b3e1fed225b76ecd6", size = 1940205, upload-time = "2026-04-07T11:14:40.319Z" }, + { url = "https://files.pythonhosted.org/packages/ea/59/b2afd98e41af9cd54554a4c1c423d84cdd60e6b1c0a09496f033b55f60ec/rapidfuzz-3.14.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6737b35d5af7479c5bf9710f7b17edd9d2c43128d974d25fb4ea653e42c64609", size = 1159639, upload-time = "2026-04-07T11:14:42.52Z" }, + { url = "https://files.pythonhosted.org/packages/a3/31/7aa7e62c4c516a7af322ed0c4f0774208b72d457d0cfec808bad0df12f4a/rapidfuzz-3.14.5-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b002c7994cc9f2bc9d9856f0fbaee6e8072c983873846c92f25cefba5b2a925f", size = 1367194, upload-time = "2026-04-07T11:14:44.25Z" }, + { url = "https://files.pythonhosted.org/packages/90/79/2fc252a63bc91d3c3b234d0a3a6ad4ebc460037a23cdcdaf9285f986e6c9/rapidfuzz-3.14.5-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:17a34330cd2a538c1ce5d400b61ba358c5b72c654b928ff87b362e88f8b864c7", size = 3151805, upload-time = "2026-04-07T11:14:46.21Z" }, + { url = "https://files.pythonhosted.org/packages/17/54/0c83508f2683ea70e2d05f8527eb07328acf7bb1e9d97a3bece5702378e7/rapidfuzz-3.14.5-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:95d937e74c1a7a1287dfb03b62a827be08ede10a155cf1af73bbf47f2b73ee6e", size = 1455667, upload-time = "2026-04-07T11:14:47.991Z" }, + { url = "https://files.pythonhosted.org/packages/71/1b/070175e873177814d58850a01ebe80e20ae11e93eb4da894d563988660fa/rapidfuzz-3.14.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:46b92a9970dcc34f0096901c792644094cab49554ac3547f35e3aebbdf0a3610", size = 2388246, upload-time = "2026-04-07T11:14:50.098Z" }, + { url = "https://files.pythonhosted.org/packages/c9/dd/77caf7aaf9c2be050ad1f128d7c24ff0f59079aa62c5f62f9df41c0af45e/rapidfuzz-3.14.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e012177c8e8a8a0754ae0d6027d63042aa5ff036d9f40f07cb3466a6082e21b8", size = 2494333, upload-time = "2026-04-07T11:14:52.303Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/dd7e1f2aa31a8fbbfc16b0610af1d770ffaf1287490f3c8c5b1c52da264f/rapidfuzz-3.14.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a2ae6f53f99c9a0eca7a0afc5b4e45fc73bc1dd4ac74c00509031d76df80ed98", size = 4258579, upload-time = "2026-04-07T11:14:54.538Z" }, + { url = "https://files.pythonhosted.org/packages/9c/0a/ac99e1ba347ba0e85e0bb60b74231d55fb93c0eff43f2920ccb413d0be08/rapidfuzz-3.14.5-cp313-cp313-win32.whl", hash = "sha256:4a60f0057231188e3bd30216f7b4e0f279b11fa4ec818bb6c1d9f014d1562fbc", size = 1709231, upload-time = "2026-04-07T11:14:56.524Z" }, + { url = "https://files.pythonhosted.org/packages/cf/cb/0e251d731b3166378644238e8f0cf9e89858c024e19f75ca9f7e3ae83fd5/rapidfuzz-3.14.5-cp313-cp313-win_amd64.whl", hash = "sha256:11bfc2ed8fbe4ab86bd516fadefab126f90e6dcadffa761739fcb304707dfd35", size = 1538519, upload-time = "2026-04-07T11:14:58.635Z" }, + { url = "https://files.pythonhosted.org/packages/30/6f/4548132acc947db6d5346a248e44a8b3a22d608ef30e770fb578caaf2d00/rapidfuzz-3.14.5-cp313-cp313-win_arm64.whl", hash = "sha256:b486b5218808f6f4dc471b114b1054e63553db69705c97da0271f47bd706aedd", size = 812628, upload-time = "2026-04-07T11:15:00.552Z" }, + { url = "https://files.pythonhosted.org/packages/00/60/69b177577290c5eab892c6f75fe89c3aff3f9ae80298a78d9372b1cecb9a/rapidfuzz-3.14.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:39ef8658aaf67d51667e7bdaf7096f432333377d8302ac43c70b5df8a4cf89b8", size = 1970231, upload-time = "2026-04-07T11:15:02.603Z" }, + { url = "https://files.pythonhosted.org/packages/48/38/2fd790052659cc4e2907b63c25433f0987864b445c1aeec1a302ef5ad948/rapidfuzz-3.14.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:9ad37a0be705b544af6296da8edddc260d10a8ae5462530fc9991f66498bb1f9", size = 1194394, upload-time = "2026-04-07T11:15:04.572Z" }, + { url = "https://files.pythonhosted.org/packages/80/f4/28430ad8472fc3536e8ebd51a864a226e979cfe924c6e3f83d111373aa74/rapidfuzz-3.14.5-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d45e06f60729e07d9b20c205f7e5cff90b6ef2584e852eecf46e045aea69627d", size = 1377051, upload-time = "2026-04-07T11:15:06.728Z" }, + { url = "https://files.pythonhosted.org/packages/77/7e/9aeacabcfd1e77397968362e5b98fe14248b8307011136b17daf99752a8e/rapidfuzz-3.14.5-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e52da10236aa6212de71b9e170bace65b64b129c0dea7fc243d6c9ce976f5074", size = 3160565, upload-time = "2026-04-07T11:15:08.667Z" }, + { url = "https://files.pythonhosted.org/packages/56/f4/db4dd7be0cd2f2022117ac5407d905f435d60e48baaea313a567ad27e865/rapidfuzz-3.14.5-cp313-cp313t-manylinux_2_39_riscv64.whl", hash = "sha256:440d30faaf682ca496170a7f0cc5453ec942e3e079f0fd802c9a7f938dfb50a3", size = 1442113, upload-time = "2026-04-07T11:15:11.138Z" }, + { url = "https://files.pythonhosted.org/packages/a4/99/0e9f6aa57f3e32a767216f797e56dc96b720fcecfb9d8ee907ecc82f8d66/rapidfuzz-3.14.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:56227a61fd3d17b0cd9793132431f3a3d07c8654be96794ba9f89fe0fc8b2d09", size = 2396618, upload-time = "2026-04-07T11:15:13.154Z" }, + { url = "https://files.pythonhosted.org/packages/60/94/44a78e39ffce17cbdd3e2b53b696acc751d5d153be0f499d052b07a4d904/rapidfuzz-3.14.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2e83cd2e25bb4edd97b689d9979d9c3acccdaaf26ceac08212ceece202febcfa", size = 2478220, upload-time = "2026-04-07T11:15:15.193Z" }, + { url = "https://files.pythonhosted.org/packages/dd/df/454311469a09a507e9d784a35796742bec22e4cebe75551e2da4e0e290fd/rapidfuzz-3.14.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:af3b859726cd3374287e405e14b9634563c078c5531a4f62375508addebddad1", size = 4265027, upload-time = "2026-04-07T11:15:17.28Z" }, + { url = "https://files.pythonhosted.org/packages/fc/01/175465a9ab3e3b70ba669058372f009d1d49c1746e2dcd56b69df188d3a5/rapidfuzz-3.14.5-cp313-cp313t-win32.whl", hash = "sha256:8ce1d850b3c0178440efde9e884d98421b5e87ff925f364d6d79e23910d7593f", size = 1766814, upload-time = "2026-04-07T11:15:19.687Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a0/a9b84a47af06ebed94a1439eb2f02adebfb8628bcd30af1fe3e02f5ef56c/rapidfuzz-3.14.5-cp313-cp313t-win_amd64.whl", hash = "sha256:c84af70bcf34e99aee894e46a0f1ac77f17d0ef828179c387407642e2466d28a", size = 1582448, upload-time = "2026-04-07T11:15:21.98Z" }, + { url = "https://files.pythonhosted.org/packages/1e/f1/5937800238b3f8248e70860d79f69ba8f73e764fff47e36bc9e2f26dbcc6/rapidfuzz-3.14.5-cp313-cp313t-win_arm64.whl", hash = "sha256:aac0ad28c686a5e72b81668b906c030ee28050b244544b8af68e12fb32543895", size = 832932, upload-time = "2026-04-07T11:15:24.358Z" }, + { url = "https://files.pythonhosted.org/packages/81/41/aa3ffb3355e62e1bf91f6599b3092e866bc88487a07c524004943c7676df/rapidfuzz-3.14.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1a31cc6d7d03e7318a0974c038959c59e19c752b81115f2e9138b3331cd64d45", size = 1943327, upload-time = "2026-04-07T11:15:26.266Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e1/c2141f1840a41e07ad2db6f724945f8f8ff3065463899a22939152dd6e09/rapidfuzz-3.14.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0298d357e2bc59d572da4db0bc631009b6f8f6c9bc8c11e99a12b833f16b6575", size = 1161755, upload-time = "2026-04-07T11:15:28.659Z" }, + { url = "https://files.pythonhosted.org/packages/ca/07/66e753eeaa353161d1d331b7dd517bb349b0bacfebe8496d7b26be26f81f/rapidfuzz-3.14.5-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:59b3dba758661a318995655435c6ab20a04ade79fa51e75bc8dc107cac8df280", size = 1376571, upload-time = "2026-04-07T11:15:31.225Z" }, + { url = "https://files.pythonhosted.org/packages/c8/85/9535df0b78ba51f478c9ce7eb6d1f85535cc31fe356773b48fd9d3e563ca/rapidfuzz-3.14.5-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4900143d82071bdda533b00300c40b14b963ff826b3642cc463b6dd0f036585e", size = 3156468, upload-time = "2026-04-07T11:15:33.428Z" }, + { url = "https://files.pythonhosted.org/packages/81/ee/b667eb93bba6dc4e0de658edd778e1619dc4d6aab68fa5e5c7f075152735/rapidfuzz-3.14.5-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:feedf219672eef83ea6be6f3bb093bba396a8560fc75be85ba225f082903df0a", size = 1458311, upload-time = "2026-04-07T11:15:35.557Z" }, + { url = "https://files.pythonhosted.org/packages/7d/ce/479074f5624364a48df3403c538797ef22d3ac49c19dc76c3f79fcdcc70c/rapidfuzz-3.14.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:419e4397a36e2665ec992d8d64c20ba4b2a42500c76ecadeca78a4f19cb9cc32", size = 2398228, upload-time = "2026-04-07T11:15:37.669Z" }, + { url = "https://files.pythonhosted.org/packages/0b/15/a8982f649150fffbdcd6f17565974501f6ab33b2795267bffbd4a7ba905b/rapidfuzz-3.14.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:97131ab2be39043054ee28d99e09efe316e6d53449b7e962dfcf3c2de8b2b246", size = 2497226, upload-time = "2026-04-07T11:15:39.857Z" }, + { url = "https://files.pythonhosted.org/packages/19/52/5267c03ef6759831b7d4625a0c9c06e87baa2fae084b61ac9c388858317b/rapidfuzz-3.14.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:593c00dac4e30231c35bf3b4f1da8ec0998762e9e94425586a5d636fcd57f9d0", size = 4262283, upload-time = "2026-04-07T11:15:42.279Z" }, + { url = "https://files.pythonhosted.org/packages/71/c0/2579f343a97f5254c43bb5853baccc01488357dcb64a27bcb869b7888a4a/rapidfuzz-3.14.5-cp314-cp314-win32.whl", hash = "sha256:0084b687b02b4e569b46d8d6d4ad25659528e6081cd6d067ca453a69035f07e4", size = 1744614, upload-time = "2026-04-07T11:15:44.498Z" }, + { url = "https://files.pythonhosted.org/packages/17/eb/8edfed1e80119dc9c35b11df4bc701eea85622ad681fff0263b6961d3224/rapidfuzz-3.14.5-cp314-cp314-win_amd64.whl", hash = "sha256:5dfa89d78f22cd773054caff44827b846161a29f2dcf7e78b8f90d086621e502", size = 1588971, upload-time = "2026-04-07T11:15:46.86Z" }, + { url = "https://files.pythonhosted.org/packages/f6/04/5676df93c85cfa57a3045d8047318df9f3cd58c7b8a99340dd95f874795e/rapidfuzz-3.14.5-cp314-cp314-win_arm64.whl", hash = "sha256:67f3f9d2b444268ab53e47d31bab89954888d23c04c6789f2c727e51fe4b1d13", size = 834985, upload-time = "2026-04-07T11:15:49.411Z" }, + { url = "https://files.pythonhosted.org/packages/f7/0d/4a8988cea658fe335048ddef8c876addff1b6daa3c9ca8ad65a5a2196e69/rapidfuzz-3.14.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:77eac0526899b3c3ad1454bb2b03cdb491d67358ec8ef0c9c48bd61b632b431d", size = 1972517, upload-time = "2026-04-07T11:15:51.819Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a3/f5cfd9965a9d9a9e32249159797c47b5d6299ea6d1629f9126b25f1c10a3/rapidfuzz-3.14.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b9c6bd754d11f6e78ac54e3d86b4b11dc1ba2f13e5fc958899574532897f5a99", size = 1196056, upload-time = "2026-04-07T11:15:54.292Z" }, + { url = "https://files.pythonhosted.org/packages/64/07/561c2e40cfd10e6630a7b0ac5a2a813aef50d944bcd1f3d260319d659d5b/rapidfuzz-3.14.5-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:738c96944d076deeaff70e92b65696ab4f7ecb8081d7791c5403a3257dfaf8ff", size = 1374732, upload-time = "2026-04-07T11:15:56.584Z" }, + { url = "https://files.pythonhosted.org/packages/c2/39/123bb94fee40e2fb3b7c49b80827c7ef42d838e18def3fc2fef5a3cf817a/rapidfuzz-3.14.5-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4c1bca487a17fe4226b4ffb2d30e799d2b274d692cffa76bd0746f56235fca3", size = 3166902, upload-time = "2026-04-07T11:15:58.768Z" }, + { url = "https://files.pythonhosted.org/packages/75/0a/45716fafc9fd2e028cf20b5ac5bc704887081cd312f84edb0e325599414b/rapidfuzz-3.14.5-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:af6a90a4ed2a48fa1a2d17e9d824e6c7c950bea5bad0b707c77fd55751e6bfef", size = 1452130, upload-time = "2026-04-07T11:16:01.453Z" }, + { url = "https://files.pythonhosted.org/packages/ca/49/4e96c413114398481c0a5b0086af32c364a18613c9a2ea578d17c4bea4ee/rapidfuzz-3.14.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bf5018938208d4597b2e679a4f8cff9fd252f1df53583130ae56281a21801b64", size = 2396308, upload-time = "2026-04-07T11:16:03.588Z" }, + { url = "https://files.pythonhosted.org/packages/89/b7/49fea9fc6878d59bd259d01dd1972d9b86117992b1c66d9b16f0a65273c3/rapidfuzz-3.14.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c0919d1f89ddf91129906705723118ea09754171e4116f5a5dbc667c7bc9b261", size = 2488210, upload-time = "2026-04-07T11:16:05.871Z" }, + { url = "https://files.pythonhosted.org/packages/0c/44/a1f732b93ffacbdad077b7c801149549b2938e1bece6addb5ad85ed74df8/rapidfuzz-3.14.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:93d8da883a35116d6813432177f35e570db5b0a5e30ecb0cbd7cb39c815735df", size = 4270621, upload-time = "2026-04-07T11:16:08.483Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ce/ff942d19fce5385054650bb71a58495ddda299d94661ccc4e6e7fa44868b/rapidfuzz-3.14.5-cp314-cp314t-win32.whl", hash = "sha256:0f23e37019ec07712d58976b1ab2b889f8649a7f7c2f626a2f34ea9139e79279", size = 1803950, upload-time = "2026-04-07T11:16:10.873Z" }, + { url = "https://files.pythonhosted.org/packages/5c/0f/9aafc63f9661222b819b391c187eed29fc90ad5935f9690e5ecc2d2047a4/rapidfuzz-3.14.5-cp314-cp314t-win_amd64.whl", hash = "sha256:7d5ca9c7832e6879a707296d1463685f7c243a27846227044504741640caec66", size = 1632357, upload-time = "2026-04-07T11:16:13.1Z" }, + { url = "https://files.pythonhosted.org/packages/70/a6/51fc1b0e61e3326e1c68a61cfd0c6b3c34c843681c4b1eefbf0596f59162/rapidfuzz-3.14.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3e91dcd2549b8f8d843f98ba03a17e01f3d8b72ce942adbbb6761bc58ffce813", size = 855409, upload-time = "2026-04-07T11:16:15.787Z" }, +] + +[[package]] +name = "regex" +version = "2026.7.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/98/04b13f1ddfb63158025291c02e03eb42fbb7acb51d091d541050eb4e35e8/regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5", size = 416440, upload-time = "2026-07-19T00:19:48.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/b9/d11d7e501ac8fd7d617684423ebb9561e0b998481c1e4cbc0cb212c5d74a/regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d", size = 496778, upload-time = "2026-07-19T00:17:05.677Z" }, + { url = "https://files.pythonhosted.org/packages/3f/a9/a5ab6f312f24318019170dc485d5421fe4f89e43a98640da50d95a8a7041/regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd", size = 297122, upload-time = "2026-07-19T00:17:07.59Z" }, + { url = "https://files.pythonhosted.org/packages/b3/63/4cab4d7f2d384a144d420b763d97674cb70619c878ea6fcd7640d0e62143/regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6", size = 292009, upload-time = "2026-07-19T00:17:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/22/85/102a81b218298957d4ea7d2f084fae537a71add9d6ff93c8e67284c5f45e/regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797", size = 796708, upload-time = "2026-07-19T00:17:11.542Z" }, + { url = "https://files.pythonhosted.org/packages/78/b5/dc136af5629938a037cd2b304c12240e132ec92f38be8ff9cc89af2a1f2d/regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18", size = 865651, upload-time = "2026-07-19T00:17:13.312Z" }, + { url = "https://files.pythonhosted.org/packages/e0/75/67402ae3cd9c8c988a4c805d15ee3eef015e7ca4cb112cf3e640fc1f4153/regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511", size = 911756, upload-time = "2026-07-19T00:17:15.015Z" }, + { url = "https://files.pythonhosted.org/packages/2a/8e/096d00c7c480ef2ff4265349b14e2261d4ab787ba1f74e2e80d1c58079c3/regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68", size = 801798, upload-time = "2026-07-19T00:17:17.208Z" }, + { url = "https://files.pythonhosted.org/packages/f0/41/e7ecac6edb5722417f85cc67eaf386322fbe8acf6918ec2fdc37c20dd9d0/regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11", size = 776933, upload-time = "2026-07-19T00:17:19.347Z" }, + { url = "https://files.pythonhosted.org/packages/6f/69/03c9b3f058d66403e0ca2c938696e81d51cd4c6d47ec5265f02f96948d9a/regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986", size = 784338, upload-time = "2026-07-19T00:17:21.057Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f7/b38ab3d43f284afbb618fcd15d0e77eb786ae461ce1f6bc7494619ddc0f2/regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b", size = 860452, upload-time = "2026-07-19T00:17:23.119Z" }, + { url = "https://files.pythonhosted.org/packages/15/5c/ff60ef0571121714f3cf9920bc183071e384a10b556d042e0fdb06cc07a5/regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb", size = 765958, upload-time = "2026-07-19T00:17:24.81Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0f/bd34021162c0ab47f9a315bd56cd5642e920c8e5668a75ef6c6a6fca590d/regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035", size = 851765, upload-time = "2026-07-19T00:17:26.993Z" }, + { url = "https://files.pythonhosted.org/packages/2a/20/a2ca43edade0595cccfdc98636739f536d9e26898e7dbddc2b9e98898953/regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a", size = 789714, upload-time = "2026-07-19T00:17:28.699Z" }, + { url = "https://files.pythonhosted.org/packages/5d/47/e02db4015d424fc83c00ea0ac8c5e5ec14397943de9abf909d5ce3a25931/regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5", size = 267157, upload-time = "2026-07-19T00:17:31.051Z" }, + { url = "https://files.pythonhosted.org/packages/08/8e/c780c131f79b42ed22d1bd7da4096c2c35f813e835acd02ef0f018bd892c/regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312", size = 277777, upload-time = "2026-07-19T00:17:32.848Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4c/e4d7e086449bdf379d89774bf1f89dc4a41943f3c5a6125a03905b34b5fb/regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d", size = 277136, upload-time = "2026-07-19T00:17:34.803Z" }, + { url = "https://files.pythonhosted.org/packages/5d/3d/84165e4299ff76f3a40fe1f2abf939e976f693383a08d2beea6af62bd2c1/regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40", size = 496552, upload-time = "2026-07-19T00:17:36.808Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/a65293e6e4cf28eb7ee1be5335a5386c40d6742e9f47fafc8fec785e16c7/regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38", size = 296983, upload-time = "2026-07-19T00:17:38.816Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/2d0564e93d87bc48618360ddca232a2ca612bbdf53ce8465d45ca5ce14ee/regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11", size = 291832, upload-time = "2026-07-19T00:17:40.726Z" }, + { url = "https://files.pythonhosted.org/packages/07/cd/42dfbabff3dfc9603c501c0e2e2c5adbb09d127b267bf5348de0af338c15/regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13", size = 796775, upload-time = "2026-07-19T00:17:42.382Z" }, + { url = "https://files.pythonhosted.org/packages/df/5d/f6a4839f2b934e3eed5973fd07f5929ee97d4c98939fb275ea23c274ee16/regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae", size = 865687, upload-time = "2026-07-19T00:17:44.185Z" }, + { url = "https://files.pythonhosted.org/packages/14/b0/b47d6c36049bc59806a50bd4c86ced70bbe058d787f80281b1d7a9b0e024/regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da", size = 911962, upload-time = "2026-07-19T00:17:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/2a/be/ff61f28f9273658cfe23acbbac5217221f6519960ed401e61dfdab12bc35/regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15", size = 801817, upload-time = "2026-07-19T00:17:48.25Z" }, + { url = "https://files.pythonhosted.org/packages/c3/bb/8b4f7f26b333f9f79e1b453613c39bb4776f51d38ae66dd0ba31d6b354ca/regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f", size = 776908, upload-time = "2026-07-19T00:17:50.183Z" }, + { url = "https://files.pythonhosted.org/packages/09/13/610110fc5921d380516d03c26b652555f08aa0d23ea78a771231873c3638/regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939", size = 784426, upload-time = "2026-07-19T00:17:52.454Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f5/1ef9e2a83a5947c57ebff0b377cb5727c3d5ec1992317a320d035cd0dbb6/regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96", size = 860600, upload-time = "2026-07-19T00:17:54.229Z" }, + { url = "https://files.pythonhosted.org/packages/a0/02/073af33a3ec149241d11c80acea91e722aa0adbf05addd50f251c4fe89c3/regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220", size = 765950, upload-time = "2026-07-19T00:17:56.041Z" }, + { url = "https://files.pythonhosted.org/packages/81/a9/d1e9f819dc394a568ef370cd56cf25394e957a2235f8370f23b576e5a475/regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc", size = 851794, upload-time = "2026-07-19T00:17:57.897Z" }, + { url = "https://files.pythonhosted.org/packages/03/3a/8ae83eda7579feacdf984e71fb9e70635fb6f832eeddca58427ec4fca926/regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2", size = 789845, upload-time = "2026-07-19T00:17:59.97Z" }, + { url = "https://files.pythonhosted.org/packages/4b/23/c195cbfe5a75fdec64d8f6554fd15237b837919d2c61bdc141d7c807b08b/regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404", size = 267135, upload-time = "2026-07-19T00:18:01.958Z" }, + { url = "https://files.pythonhosted.org/packages/b2/80/a11de8404b7272b70acb45c1c05987cce60b45d5693da2e176f0e390d564/regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e", size = 277747, upload-time = "2026-07-19T00:18:04.121Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/0f5c8eff1b4f1f3d83276d365fccecf666afcc7d947420943bf394d07adb/regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8", size = 277129, upload-time = "2026-07-19T00:18:05.821Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4c/44b74742052cedda40f9ae469532a037112f7311a36669a891fba8984bb0/regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2", size = 501134, upload-time = "2026-07-19T00:18:07.567Z" }, + { url = "https://files.pythonhosted.org/packages/f0/45/bbd038b5e39ee5613a5a689290145b40058cc152c41de9cc23639d2b9734/regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda", size = 299418, upload-time = "2026-07-19T00:18:09.38Z" }, + { url = "https://files.pythonhosted.org/packages/65/38/c5bde94b4cedfd5850d64c3f08222d8e1600e84f6ee71d9b44b4b8163f74/regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff", size = 294486, upload-time = "2026-07-19T00:18:11.188Z" }, + { url = "https://files.pythonhosted.org/packages/d7/6a/2f5e107cb26c960b781967178899daf2787a7ab151844ed3c01d6fc95474/regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1", size = 811643, upload-time = "2026-07-19T00:18:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/37/d4/a2f963406d7d73a62eed84ba05a258afb6cad1b21aa4517443ce40506b78/regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf", size = 871081, upload-time = "2026-07-19T00:18:14.733Z" }, + { url = "https://files.pythonhosted.org/packages/45/a3/44be546340bedb15f13063f5e7fe16793ea4d9ea2e805d09bd174ac27724/regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732", size = 917372, upload-time = "2026-07-19T00:18:16.724Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f6/e0870b0fd2a40dba0074e4b76e514b21313d37946c9248453e34ec43923e/regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a", size = 816089, upload-time = "2026-07-19T00:18:18.617Z" }, + { url = "https://files.pythonhosted.org/packages/ae/27/957e8e22690ad6634572b39b71f130a6105f4d0718bb16849eac00fff147/regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba", size = 785206, upload-time = "2026-07-19T00:18:20.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/a4/186e410941e731037c01166069ab86da9f65e8f8110c18009ccf4bd623ee/regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc", size = 800431, upload-time = "2026-07-19T00:18:22.716Z" }, + { url = "https://files.pythonhosted.org/packages/73/9f/e4e10e023d291d64a33e246610b724493bf1ce98e0e59c9b7c837e5acfb7/regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62", size = 864906, upload-time = "2026-07-19T00:18:24.772Z" }, + { url = "https://files.pythonhosted.org/packages/24/57/ccb20b6be5f1f52a053d1ba2a8f7a077edb9d918248b8490d7506c6832b3/regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1", size = 773559, upload-time = "2026-07-19T00:18:27.008Z" }, + { url = "https://files.pythonhosted.org/packages/a3/82/f3b263cf8fad927dc102891da8502e718b7ff9d19af7a2a07c03865d7188/regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e", size = 857739, upload-time = "2026-07-19T00:18:29.107Z" }, + { url = "https://files.pythonhosted.org/packages/47/2e/1687bd1b6c2aed5e672ccf845fc11557821fe7366d921b50889ea5ce57bf/regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0", size = 804522, upload-time = "2026-07-19T00:18:31.362Z" }, + { url = "https://files.pythonhosted.org/packages/76/7c/cc4e7655181b2d9235b704f2c5e19d8eff002bbc437bae59baee0e381aca/regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4", size = 269141, upload-time = "2026-07-19T00:18:33.479Z" }, + { url = "https://files.pythonhosted.org/packages/bb/14/961b4c7b05a2391c32dbc85e27773076671ef8f97f36cec70fe414734c02/regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974", size = 280036, upload-time = "2026-07-19T00:18:35.419Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/795644550d788ddbb6dc458c95895f8009978ea6d6ea76b005eb3f45e8c9/regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d", size = 279394, upload-time = "2026-07-19T00:18:37.454Z" }, + { url = "https://files.pythonhosted.org/packages/d2/25/0c4c452f8ef3efe456745b2f33195f5904b573fb4c2ff3f0cb9ec188461e/regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd", size = 496750, upload-time = "2026-07-19T00:18:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/24/9e/b70ca6c1704f6c7cd32a9e143c86cc5968d10981eca284bad670c245ea7d/regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac", size = 297093, upload-time = "2026-07-19T00:18:41.583Z" }, + { url = "https://files.pythonhosted.org/packages/87/74/0b692da2520d51fbff19c88b83d97e4c702909dd02386c585998b7e2dbed/regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5", size = 292043, upload-time = "2026-07-19T00:18:43.347Z" }, + { url = "https://files.pythonhosted.org/packages/e3/a7/1d478e614016045a33feae57446215f9fd65b665a5ceb2f891fb3183bc52/regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3", size = 797214, upload-time = "2026-07-19T00:18:45.362Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ae/11b9c9411d92c30e3d2db32df5a31133e4a99a8fc397a604fd08f6c4bffb/regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053", size = 866433, upload-time = "2026-07-19T00:18:47.315Z" }, + { url = "https://files.pythonhosted.org/packages/b1/62/2b2efc4992f91d6d204b24c647c9f9412e85379d92b7c0ab9fdae622327e/regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b", size = 911360, upload-time = "2026-07-19T00:18:49.588Z" }, + { url = "https://files.pythonhosted.org/packages/14/71/986ceea9aa3da548bf1357cad89b63915ec6d21ec957c8113b29ece567df/regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a", size = 801275, upload-time = "2026-07-19T00:18:51.767Z" }, + { url = "https://files.pythonhosted.org/packages/15/be/ce9d9534b2cda96eab32c548261224b9b4e220a4126f098f60f42ae7b4cd/regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1", size = 777131, upload-time = "2026-07-19T00:18:54.053Z" }, + { url = "https://files.pythonhosted.org/packages/61/2b/58b5c710f2c3929515a25f3a1ca0dad0dcd4518d4fff3cf23bc7adb8dcd2/regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e", size = 785020, upload-time = "2026-07-19T00:18:56.579Z" }, + { url = "https://files.pythonhosted.org/packages/84/03/5fe091935b74f15fe0f97998c215cae418d1c0413f6258c7d4d2e83aa37f/regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12", size = 861263, upload-time = "2026-07-19T00:18:58.64Z" }, + { url = "https://files.pythonhosted.org/packages/d8/fa/d60bf82e10841eef62a9e32aac401468f05fddfbcb2942e342b1ba3d2433/regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2", size = 766199, upload-time = "2026-07-19T00:19:00.705Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5d/11e64d151b0662b81d6bf644c74dc118d461df85bdf2577fadbbf751788a/regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97", size = 851317, upload-time = "2026-07-19T00:19:03.015Z" }, + { url = "https://files.pythonhosted.org/packages/7c/34/532efb87488d90807bae6a443d357ee5e2728a478c597619c8aaa17cc0bd/regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4", size = 789557, upload-time = "2026-07-19T00:19:05.338Z" }, + { url = "https://files.pythonhosted.org/packages/d6/90/3a8d5ca977171ec3ae21a71207d2228b2663bde14d7f7ef0e6363ecf9290/regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa", size = 272531, upload-time = "2026-07-19T00:19:07.487Z" }, + { url = "https://files.pythonhosted.org/packages/96/e1/8862885e70409de70e8c005f57fb2e7be8d9ef0317250d60f4c9660a300d/regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac", size = 280831, upload-time = "2026-07-19T00:19:09.46Z" }, + { url = "https://files.pythonhosted.org/packages/08/82/2693e53e29f9104d9de95d37ce4dd826bd32d5f9c0085d3aa6ac042675c4/regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459", size = 281099, upload-time = "2026-07-19T00:19:11.398Z" }, + { url = "https://files.pythonhosted.org/packages/92/b7/9a01aa16461a18cde9d7b9c3ab21e501db2ce33725f53014342b91df2b0a/regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3", size = 501121, upload-time = "2026-07-19T00:19:13.425Z" }, + { url = "https://files.pythonhosted.org/packages/f3/5e/bbaeca815dc9191c424c94a4fdc5c87c75748a64a6271821212ebdd4e1a3/regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518", size = 299415, upload-time = "2026-07-19T00:19:15.43Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d6/0dd1a321afaab95eb7ff44aa0f637301786f1dc71c6b797b9ed236ed8890/regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9", size = 294483, upload-time = "2026-07-19T00:19:17.879Z" }, + { url = "https://files.pythonhosted.org/packages/92/5f/40bacf91d0904f812e13bbbab3864604c463eced8afdc54aeaa50492ea95/regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435", size = 811833, upload-time = "2026-07-19T00:19:20.102Z" }, + { url = "https://files.pythonhosted.org/packages/94/7c/4902744261f775aeede8b5627314b38482da29cf49a57b66a6fb753246c5/regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0", size = 871270, upload-time = "2026-07-19T00:19:22.365Z" }, + { url = "https://files.pythonhosted.org/packages/16/70/6980c9be6bf21c0a60ed3e0aea39cf419ecf3b08d1d9947bc56e196ef186/regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a", size = 917534, upload-time = "2026-07-19T00:19:24.529Z" }, + { url = "https://files.pythonhosted.org/packages/52/92/8b2bd872782ce8c42691e39acb38eb8efe014e5ddb78ad7d943d6f197ce9/regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276", size = 816135, upload-time = "2026-07-19T00:19:26.919Z" }, + { url = "https://files.pythonhosted.org/packages/de/2d/33a602f657bdc4041f17d79f92ab18261d255d91a06117a6e29df023e5e2/regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c", size = 785492, upload-time = "2026-07-19T00:19:29.192Z" }, + { url = "https://files.pythonhosted.org/packages/9e/36/0987cf4cb271680064a70d24a475873775a151d0b7058698a006cb0cae4a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a", size = 800658, upload-time = "2026-07-19T00:19:31.392Z" }, + { url = "https://files.pythonhosted.org/packages/a8/24/c14f31c135e1ba55fa4f9a58ca98d0842512bf6188230763c31c8f449e3b/regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009", size = 865073, upload-time = "2026-07-19T00:19:33.485Z" }, + { url = "https://files.pythonhosted.org/packages/14/85/181a12211f22469f24d2de1ebddfe397d2396e2c29013b9a58134a91069a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218", size = 773684, upload-time = "2026-07-19T00:19:35.599Z" }, + { url = "https://files.pythonhosted.org/packages/23/58/bd1a0c1a62251366f8d21f41b1ea3c76994962071b8b6ea42f72d505c0f0/regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966", size = 857769, upload-time = "2026-07-19T00:19:37.738Z" }, + { url = "https://files.pythonhosted.org/packages/e4/4f/f7e2dad6756b2fe1fe75dd90a628c3b45f249d39f948dd90cd2476325417/regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44", size = 804546, upload-time = "2026-07-19T00:19:40.229Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d7/01d31d5bdb09bc026fab77f59a371fdf8f9b292e4810546c56182ca70498/regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78", size = 274526, upload-time = "2026-07-19T00:19:42.398Z" }, + { url = "https://files.pythonhosted.org/packages/52/0e/cea4ce73bc0a8247a0748228ae6669984c7e1f8134b6fa66e59c0572e0ea/regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2", size = 283763, upload-time = "2026-07-19T00:19:44.644Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b6/26e41975febae63b7a6e3e02f32cff6cff2e4f10d19c929082f56aebf7c6/regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547", size = 283451, upload-time = "2026-07-19T00:19:46.639Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[package.optional-dependencies] +socks = [ + { name = "pysocks" }, +] + +[[package]] +name = "requests-toolbelt" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888, upload-time = "2023-05-01T04:11:33.229Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, +] + +[[package]] +name = "rich" +version = "13.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ab/3a/0316b28d0761c6734d6bc14e770d85506c986c85ffb239e688eeaab2c2bc/rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098", size = 223149, upload-time = "2024-11-01T16:43:57.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/71/39c7c0d87f8d4e6c020a393182060eaefeeae6c01dab6a84ec346f2567df/rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90", size = 242424, upload-time = "2024-11-01T16:43:55.817Z" }, +] + +[[package]] +name = "ruff" +version = "0.16.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/70/25/7113f6d5498888c5fb7db34081cba7d5971c4cb1bfb26819966eee68f003/ruff-0.16.1.tar.gz", hash = "sha256:fedad7c801dabd3fb9741d76aca39246e6ddd9ca446a015875207bf19f1e6bc7", size = 4877500, upload-time = "2026-07-30T19:37:01.379Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/bd/694da69368e0973de65df2ddc73ab18d43c469d5963d9b150911de6bc513/ruff-0.16.1-py3-none-linux_armv6l.whl", hash = "sha256:58edb313b88f0c5460a26adf5f39a37a3be789494a15e3e411e35fa78b89f9a0", size = 10839126, upload-time = "2026-07-30T19:36:13.697Z" }, + { url = "https://files.pythonhosted.org/packages/3f/f0/b626e5d5bd0dd9576263658ef12885e2288afd1029a48e26ffed65ec1ac1/ruff-0.16.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fde5a99e2f97479af66edd6622c6d5a2a7592c77cf4153d9e4428f5eeb55b60c", size = 11070253, upload-time = "2026-07-30T19:36:17.14Z" }, + { url = "https://files.pythonhosted.org/packages/83/63/f40acfb6b35b88623e71684942b552c3edd96035f5d98f313815f7b277de/ruff-0.16.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e0d4c20532fca4f7fa609369161d968dd28f65d83dabbd61d8e9c7edbf7001f6", size = 10561425, upload-time = "2026-07-30T19:36:20.04Z" }, + { url = "https://files.pythonhosted.org/packages/aa/dd/14ec0e9c2b4d315547dd38765004b4863e354e1b52cb308272215d9f6f6d/ruff-0.16.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30affbcedf59ad5703d9c91f82266e02b47739f797e1a7b6e158e5526a6dae38", size = 10948879, upload-time = "2026-07-30T19:36:22.476Z" }, + { url = "https://files.pythonhosted.org/packages/33/e9/9d870cbae575030fdef595f04b4b97573c525b5497cce4f4498cf2f85446/ruff-0.16.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24e9c631573cbca9d20f1283f8f479b2afa4a8503504822bd71a293889f16743", size = 10643691, upload-time = "2026-07-30T19:36:24.914Z" }, + { url = "https://files.pythonhosted.org/packages/c4/09/12743d544e2173f53ecd27217c65f90d2bc0f8424a66a60339e56bbc0457/ruff-0.16.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b41bdd48fb420987a9b5212e4957c26ad4abce401fa9ea9d4d85843727945f4f", size = 11435354, upload-time = "2026-07-30T19:36:28.447Z" }, + { url = "https://files.pythonhosted.org/packages/7f/89/a1652b2daee52083c9554a6333b678a8b01d0400f976827bb87857f9449a/ruff-0.16.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b0d1e1393b7648079e13669de1c1f4fde06d4583e84d8fd5c1551e0a77a2aa75", size = 12259033, upload-time = "2026-07-30T19:36:31.326Z" }, + { url = "https://files.pythonhosted.org/packages/16/96/ecdcb8c54ee7b123b487f807eb014e6e019155a0b81dfb669acd52f28ce3/ruff-0.16.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07bf434b1c95f4e093be4532068ef4fcf00924eb2ade8796075980902d6fd54a", size = 11667981, upload-time = "2026-07-30T19:36:34.394Z" }, + { url = "https://files.pythonhosted.org/packages/cd/90/c52e12e0d862e9572f2a33aa227409143520abe53111e9a6babbac7b4af8/ruff-0.16.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39897739f112253ee4fdd2e8aa9a4f9ded99fb2be367d5f31dfa4ded6025584c", size = 11468183, upload-time = "2026-07-30T19:36:37.339Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6b/4ffb7ad1d83eb16cf8cbb3c8815d3f11c88460fd162d4b372a2059be1c2a/ruff-0.16.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:82ae3c0c0d74daf17b968a10b7b3bb3ef297ab7de0c1f749646b25e690ccb150", size = 11470071, upload-time = "2026-07-30T19:36:39.91Z" }, + { url = "https://files.pythonhosted.org/packages/9c/72/32ae7db4c0b5e32ab611787caa19d1546800676d79f7483b7100a3561bf4/ruff-0.16.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4d5f2ed10f8242d83fc08d521301089364e3375375705356f20c0e31606ef3ef", size = 10919503, upload-time = "2026-07-30T19:36:42.65Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ca/3d901ba6ad6fc38da39c3448fc6c59ac945679293a17c3ceb6d6c1cba13e/ruff-0.16.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a4665b309891f83f3e3c25447935f1213e9abbd4b5640af7a1f2def9f8d413c1", size = 10649861, upload-time = "2026-07-30T19:36:45.18Z" }, + { url = "https://files.pythonhosted.org/packages/92/79/894ef1ced26552d5f8c9cf6d85b0687840e1128c55aeab7b9c2d54a0d880/ruff-0.16.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:26e9ca5c9bc3971f20d3cf18a957f52ffd6a5f6564ff15c4912a144dcac22494", size = 11148137, upload-time = "2026-07-30T19:36:47.936Z" }, + { url = "https://files.pythonhosted.org/packages/2d/69/3609a09fa1cb46cc28b762363e440a354204e5dff01bd0c8d7437874d6b9/ruff-0.16.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:67e1e1e3fa4f0c82f0e36d4cd61e661f6e7a6196cb1aa92fe0828fa7b8f257cd", size = 11559211, upload-time = "2026-07-30T19:36:50.448Z" }, + { url = "https://files.pythonhosted.org/packages/fc/8a/fb22af2fd78a736e241fabf67e30ce1799a64244026377a49e133af90762/ruff-0.16.1-py3-none-win32.whl", hash = "sha256:d31765e131295b8445caf301e3e8a85b34d1b9b211b4109b7ba457888b051806", size = 10838258, upload-time = "2026-07-30T19:36:53.298Z" }, + { url = "https://files.pythonhosted.org/packages/d4/35/e57fd9fb5d423961df087a00b12d42c0a830288dc2f3b45ecca299158b4f/ruff-0.16.1-py3-none-win_amd64.whl", hash = "sha256:09b05e8b90c2cb06ad63464350e7a45e8e44a2dfe52072ebfba6666ca8d3f596", size = 11961111, upload-time = "2026-07-30T19:36:56.107Z" }, + { url = "https://files.pythonhosted.org/packages/cb/46/240ea004bf6dc4feb40e9832f2205a476a47dd5b8a3f8211a5fc5f95e20e/ruff-0.16.1-py3-none-win_arm64.whl", hash = "sha256:dbaadaac38c70239f056d306b7476f246b0bf000fa6b3876402acbf5b227eaf8", size = 11309414, upload-time = "2026-07-30T19:36:58.79Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.19.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/e7/5c595c75e9f41a44f30e526eda465ea0b4eec93470e074e4a111b253f13a/s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25", size = 90216, upload-time = "2026-07-22T19:30:43.251Z" }, +] + +[[package]] +name = "scikit-image" +version = "0.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "imageio" }, + { name = "lazy-loader" }, + { name = "networkx" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "scipy" }, + { name = "tifffile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/b4/2528bb43c67d48053a7a649a9666432dc307d66ba02e3a6d5c40f46655df/scikit_image-0.26.0.tar.gz", hash = "sha256:f5f970ab04efad85c24714321fcc91613fcb64ef2a892a13167df2f3e59199fa", size = 22729739, upload-time = "2025-12-20T17:12:21.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/e8/e13757982264b33a1621628f86b587e9a73a13f5256dad49b19ba7dc9083/scikit_image-0.26.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d454b93a6fa770ac5ae2d33570f8e7a321bb80d29511ce4b6b78058ebe176e8c", size = 12376452, upload-time = "2025-12-20T17:10:52.796Z" }, + { url = "https://files.pythonhosted.org/packages/e3/be/f8dd17d0510f9911f9f17ba301f7455328bf13dae416560126d428de9568/scikit_image-0.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3409e89d66eff5734cd2b672d1c48d2759360057e714e1d92a11df82c87cba37", size = 12061567, upload-time = "2025-12-20T17:10:55.207Z" }, + { url = "https://files.pythonhosted.org/packages/b3/2b/c70120a6880579fb42b91567ad79feb4772f7be72e8d52fec403a3dde0c6/scikit_image-0.26.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c717490cec9e276afb0438dd165b7c3072d6c416709cc0f9f5a4c1070d23a44", size = 13084214, upload-time = "2025-12-20T17:10:57.468Z" }, + { url = "https://files.pythonhosted.org/packages/f4/a2/70401a107d6d7466d64b466927e6b96fcefa99d57494b972608e2f8be50f/scikit_image-0.26.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7df650e79031634ac90b11e64a9eedaf5a5e06fcd09bcd03a34be01745744466", size = 13561683, upload-time = "2025-12-20T17:10:59.49Z" }, + { url = "https://files.pythonhosted.org/packages/13/a5/48bdfd92794c5002d664e0910a349d0a1504671ef5ad358150f21643c79a/scikit_image-0.26.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cefd85033e66d4ea35b525bb0937d7f42d4cdcfed2d1888e1570d5ce450d3932", size = 14112147, upload-time = "2025-12-20T17:11:02.083Z" }, + { url = "https://files.pythonhosted.org/packages/ee/b5/ac71694da92f5def5953ca99f18a10fe98eac2dd0a34079389b70b4d0394/scikit_image-0.26.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3f5bf622d7c0435884e1e141ebbe4b2804e16b2dd23ae4c6183e2ea99233be70", size = 14661625, upload-time = "2025-12-20T17:11:04.528Z" }, + { url = "https://files.pythonhosted.org/packages/23/4d/a3cc1e96f080e253dad2251bfae7587cf2b7912bcd76fd43fd366ff35a87/scikit_image-0.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:abed017474593cd3056ae0fe948d07d0747b27a085e92df5474f4955dd65aec0", size = 11911059, upload-time = "2025-12-20T17:11:06.61Z" }, + { url = "https://files.pythonhosted.org/packages/35/8a/d1b8055f584acc937478abf4550d122936f420352422a1a625eef2c605d8/scikit_image-0.26.0-cp312-cp312-win_arm64.whl", hash = "sha256:4d57e39ef67a95d26860c8caf9b14b8fb130f83b34c6656a77f191fa6d1d04d8", size = 11348740, upload-time = "2025-12-20T17:11:09.118Z" }, + { url = "https://files.pythonhosted.org/packages/4f/48/02357ffb2cca35640f33f2cfe054a4d6d5d7a229b88880a64f1e45c11f4e/scikit_image-0.26.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a2e852eccf41d2d322b8e60144e124802873a92b8d43a6f96331aa42888491c7", size = 12346329, upload-time = "2025-12-20T17:11:11.599Z" }, + { url = "https://files.pythonhosted.org/packages/67/b9/b792c577cea2c1e94cda83b135a656924fc57c428e8a6d302cd69aac1b60/scikit_image-0.26.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:98329aab3bc87db352b9887f64ce8cdb8e75f7c2daa19927f2e121b797b678d5", size = 12031726, upload-time = "2025-12-20T17:11:13.871Z" }, + { url = "https://files.pythonhosted.org/packages/07/a9/9564250dfd65cb20404a611016db52afc6268b2b371cd19c7538ea47580f/scikit_image-0.26.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:915bb3ba66455cf8adac00dc8fdf18a4cd29656aec7ddd38cb4dda90289a6f21", size = 13094910, upload-time = "2025-12-20T17:11:16.2Z" }, + { url = "https://files.pythonhosted.org/packages/a3/b8/0d8eeb5a9fd7d34ba84f8a55753a0a3e2b5b51b2a5a0ade648a8db4a62f7/scikit_image-0.26.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b36ab5e778bf50af5ff386c3ac508027dc3aaeccf2161bdf96bde6848f44d21b", size = 13660939, upload-time = "2025-12-20T17:11:18.464Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d6/91d8973584d4793d4c1a847d388e34ef1218d835eeddecfc9108d735b467/scikit_image-0.26.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:09bad6a5d5949c7896c8347424c4cca899f1d11668030e5548813ab9c2865dcb", size = 14138938, upload-time = "2025-12-20T17:11:20.919Z" }, + { url = "https://files.pythonhosted.org/packages/39/9a/7e15d8dc10d6bbf212195fb39bdeb7f226c46dd53f9c63c312e111e2e175/scikit_image-0.26.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:aeb14db1ed09ad4bee4ceb9e635547a8d5f3549be67fc6c768c7f923e027e6cd", size = 14752243, upload-time = "2025-12-20T17:11:23.347Z" }, + { url = "https://files.pythonhosted.org/packages/8f/58/2b11b933097bc427e42b4a8b15f7de8f24f2bac1fd2779d2aea1431b2c31/scikit_image-0.26.0-cp313-cp313-win_amd64.whl", hash = "sha256:ac529eb9dbd5954f9aaa2e3fe9a3fd9661bfe24e134c688587d811a0233127f1", size = 11906770, upload-time = "2025-12-20T17:11:25.297Z" }, + { url = "https://files.pythonhosted.org/packages/ad/ec/96941474a18a04b69b6f6562a5bd79bd68049fa3728d3b350976eccb8b93/scikit_image-0.26.0-cp313-cp313-win_arm64.whl", hash = "sha256:a2d211bc355f59725efdcae699b93b30348a19416cc9e017f7b2fb599faf7219", size = 11342506, upload-time = "2025-12-20T17:11:27.399Z" }, + { url = "https://files.pythonhosted.org/packages/03/e5/c1a9962b0cf1952f42d32b4a2e48eed520320dbc4d2ff0b981c6fa508b6b/scikit_image-0.26.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:9eefb4adad066da408a7601c4c24b07af3b472d90e08c3e7483d4e9e829d8c49", size = 12663278, upload-time = "2025-12-20T17:11:29.358Z" }, + { url = "https://files.pythonhosted.org/packages/ae/97/c1a276a59ce8e4e24482d65c1a3940d69c6b3873279193b7ebd04e5ee56b/scikit_image-0.26.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:6caec76e16c970c528d15d1c757363334d5cb3069f9cea93d2bead31820511f3", size = 12405142, upload-time = "2025-12-20T17:11:31.282Z" }, + { url = "https://files.pythonhosted.org/packages/d4/4a/f1cbd1357caef6c7993f7efd514d6e53d8fd6f7fe01c4714d51614c53289/scikit_image-0.26.0-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a07200fe09b9d99fcdab959859fe0f7db8df6333d6204344425d476850ce3604", size = 12942086, upload-time = "2025-12-20T17:11:33.683Z" }, + { url = "https://files.pythonhosted.org/packages/5b/6f/74d9fb87c5655bd64cf00b0c44dc3d6206d9002e5f6ba1c9aeb13236f6bf/scikit_image-0.26.0-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92242351bccf391fc5df2d1529d15470019496d2498d615beb68da85fe7fdf37", size = 13265667, upload-time = "2025-12-20T17:11:36.11Z" }, + { url = "https://files.pythonhosted.org/packages/a7/73/faddc2413ae98d863f6fa2e3e14da4467dd38e788e1c23346cf1a2b06b97/scikit_image-0.26.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:52c496f75a7e45844d951557f13c08c81487c6a1da2e3c9c8a39fcde958e02cc", size = 14001966, upload-time = "2025-12-20T17:11:38.55Z" }, + { url = "https://files.pythonhosted.org/packages/02/94/9f46966fa042b5d57c8cd641045372b4e0df0047dd400e77ea9952674110/scikit_image-0.26.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:20ef4a155e2e78b8ab973998e04d8a361d49d719e65412405f4dadd9155a61d9", size = 14359526, upload-time = "2025-12-20T17:11:41.087Z" }, + { url = "https://files.pythonhosted.org/packages/5d/b4/2840fe38f10057f40b1c9f8fb98a187a370936bf144a4ac23452c5ef1baf/scikit_image-0.26.0-cp313-cp313t-win_amd64.whl", hash = "sha256:c9087cf7d0e7f33ab5c46d2068d86d785e70b05400a891f73a13400f1e1faf6a", size = 12287629, upload-time = "2025-12-20T17:11:43.11Z" }, + { url = "https://files.pythonhosted.org/packages/22/ba/73b6ca70796e71f83ab222690e35a79612f0117e5aaf167151b7d46f5f2c/scikit_image-0.26.0-cp313-cp313t-win_arm64.whl", hash = "sha256:27d58bc8b2acd351f972c6508c1b557cfed80299826080a4d803dd29c51b707e", size = 11647755, upload-time = "2025-12-20T17:11:45.279Z" }, + { url = "https://files.pythonhosted.org/packages/51/44/6b744f92b37ae2833fd423cce8f806d2368859ec325a699dc30389e090b9/scikit_image-0.26.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:63af3d3a26125f796f01052052f86806da5b5e54c6abef152edb752683075a9c", size = 12365810, upload-time = "2025-12-20T17:11:47.357Z" }, + { url = "https://files.pythonhosted.org/packages/40/f5/83590d9355191f86ac663420fec741b82cc547a4afe7c4c1d986bf46e4db/scikit_image-0.26.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ce00600cd70d4562ed59f80523e18cdcc1fae0e10676498a01f73c255774aefd", size = 12075717, upload-time = "2025-12-20T17:11:49.483Z" }, + { url = "https://files.pythonhosted.org/packages/72/48/253e7cf5aee6190459fe136c614e2cbccc562deceb4af96e0863f1b8ee29/scikit_image-0.26.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6381edf972b32e4f54085449afde64365a57316637496c1325a736987083e2ab", size = 13161520, upload-time = "2025-12-20T17:11:51.58Z" }, + { url = "https://files.pythonhosted.org/packages/73/c3/cec6a3cbaadfdcc02bd6ff02f3abfe09eaa7f4d4e0a525a1e3a3f4bce49c/scikit_image-0.26.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6624a76c6085218248154cc7e1500e6b488edcd9499004dd0d35040607d7505", size = 13684340, upload-time = "2025-12-20T17:11:53.708Z" }, + { url = "https://files.pythonhosted.org/packages/d4/0d/39a776f675d24164b3a267aa0db9f677a4cb20127660d8bf4fd7fef66817/scikit_image-0.26.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f775f0e420faac9c2aa6757135f4eb468fb7b70e0b67fa77a5e79be3c30ee331", size = 14203839, upload-time = "2025-12-20T17:11:55.89Z" }, + { url = "https://files.pythonhosted.org/packages/ee/25/2514df226bbcedfe9b2caafa1ba7bc87231a0c339066981b182b08340e06/scikit_image-0.26.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede4d6d255cc5da9faeb2f9ba7fedbc990abbc652db429f40a16b22e770bb578", size = 14770021, upload-time = "2025-12-20T17:11:58.014Z" }, + { url = "https://files.pythonhosted.org/packages/8d/5b/0671dc91c0c79340c3fe202f0549c7d3681eb7640fe34ab68a5f090a7c7f/scikit_image-0.26.0-cp314-cp314-win_amd64.whl", hash = "sha256:0660b83968c15293fd9135e8d860053ee19500d52bf55ca4fb09de595a1af650", size = 12023490, upload-time = "2025-12-20T17:12:00.013Z" }, + { url = "https://files.pythonhosted.org/packages/65/08/7c4cb59f91721f3de07719085212a0b3962e3e3f2d1818cbac4eeb1ea53e/scikit_image-0.26.0-cp314-cp314-win_arm64.whl", hash = "sha256:b8d14d3181c21c11170477a42542c1addc7072a90b986675a71266ad17abc37f", size = 11473782, upload-time = "2025-12-20T17:12:01.983Z" }, + { url = "https://files.pythonhosted.org/packages/49/41/65c4258137acef3d73cb561ac55512eacd7b30bb4f4a11474cad526bc5db/scikit_image-0.26.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:cde0bbd57e6795eba83cb10f71a677f7239271121dc950bc060482834a668ad1", size = 12686060, upload-time = "2025-12-20T17:12:03.886Z" }, + { url = "https://files.pythonhosted.org/packages/e7/32/76971f8727b87f1420a962406388a50e26667c31756126444baf6668f559/scikit_image-0.26.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:163e9afb5b879562b9aeda0dd45208a35316f26cc7a3aed54fd601604e5cf46f", size = 12422628, upload-time = "2025-12-20T17:12:05.921Z" }, + { url = "https://files.pythonhosted.org/packages/37/0d/996febd39f757c40ee7b01cdb861867327e5c8e5f595a634e8201462d958/scikit_image-0.26.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:724f79fd9b6cb6f4a37864fe09f81f9f5d5b9646b6868109e1b100d1a7019e59", size = 12962369, upload-time = "2025-12-20T17:12:07.912Z" }, + { url = "https://files.pythonhosted.org/packages/48/b4/612d354f946c9600e7dea012723c11d47e8d455384e530f6daaaeb9bf62c/scikit_image-0.26.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3268f13310e6857508bd87202620df996199a016a1d281b309441d227c822394", size = 13272431, upload-time = "2025-12-20T17:12:10.255Z" }, + { url = "https://files.pythonhosted.org/packages/0a/6e/26c00b466e06055a086de2c6e2145fe189ccdc9a1d11ccc7de020f2591ad/scikit_image-0.26.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fac96a1f9b06cd771cbbb3cd96c5332f36d4efd839b1d8b053f79e5887acde62", size = 14016362, upload-time = "2025-12-20T17:12:12.793Z" }, + { url = "https://files.pythonhosted.org/packages/47/88/00a90402e1775634043c2a0af8a3c76ad450866d9fa444efcc43b553ba2d/scikit_image-0.26.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2c1e7bd342f43e7a97e571b3f03ba4c1293ea1a35c3f13f41efdc8a81c1dc8f2", size = 14364151, upload-time = "2025-12-20T17:12:14.909Z" }, + { url = "https://files.pythonhosted.org/packages/da/ca/918d8d306bd43beacff3b835c6d96fac0ae64c0857092f068b88db531a7c/scikit_image-0.26.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b702c3bb115e1dcf4abf5297429b5c90f2189655888cbed14921f3d26f81d3a4", size = 12413484, upload-time = "2025-12-20T17:12:17.046Z" }, + { url = "https://files.pythonhosted.org/packages/dc/cd/4da01329b5a8d47ff7ec3c99a2b02465a8017b186027590dc7425cee0b56/scikit_image-0.26.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0608aa4a9ec39e0843de10d60edb2785a30c1c47819b67866dd223ebd149acaf", size = 11769501, upload-time = "2025-12-20T17:12:19.339Z" }, +] + +[[package]] +name = "scipy" +version = "1.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/19/ca10ead60b0acc80b2b833c2c4a4f2ff753d0f58b811f70d911c7e94a25c/scipy-1.18.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:7bd21faaf5a1a3b2eff922d02db5f191b99a6518db9078a8fb23169f6d22259a", size = 31056519, upload-time = "2026-06-19T14:59:45.203Z" }, + { url = "https://files.pythonhosted.org/packages/96/72/1e6442a00cd2924d361aa1b642ab6373ec35c6fabf311a760be9f76e0f13/scipy-1.18.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:265915e79107de9f946b855e50d7470d5893ec3f54b342e1aa6201cbdcd8bb6b", size = 28681889, upload-time = "2026-06-19T14:59:48.103Z" }, + { url = "https://files.pythonhosted.org/packages/9b/2d/11dd93d21e147a73ba22bd75c0b9208d3a2e0ec76d53170ce7d9029b1015/scipy-1.18.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9ab7b758be6940954a713ee466e2043e9f6e2ed965c1fce5c91039f4be3d90a9", size = 20423580, upload-time = "2026-06-19T14:59:50.665Z" }, + { url = "https://files.pythonhosted.org/packages/9c/01/93552f75e0d2a7dd115a45e59209c51e8d514daff02fc887d2623be06fe1/scipy-1.18.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:97b6cddaaee0a779ef6b5ca83c9604b27cc16b2b8fc22c142652df8793319fb8", size = 23054441, upload-time = "2026-06-19T14:59:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/3c/23/21f5e703643d66f21faa6b4c73195bfcad70c55efcb4f1ab327cd7c4101a/scipy-1.18.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:52a96e21517c7292375c0e27dd796a811f03fcea5fd4d108fdfea8145dcf17ab", size = 33968720, upload-time = "2026-06-19T14:59:56.415Z" }, + { url = "https://files.pythonhosted.org/packages/dd/aa/1b939f6c67ed68635bb538e6752d3dacc02f66535182e939a89581a44e9c/scipy-1.18.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f55797419e16e7f30cf88ffb3113ce0467f00cfe3f70d5c281730b21769bfc2", size = 35287115, upload-time = "2026-06-19T14:59:59.411Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ff/eec46be7e9234208f801062b53e1983085eddebd693f6c9bfb03b459830d/scipy-1.18.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ad033410e2e0672ffdc1042110cef20e1c46f8fd0616cee1d44d8d58fad8fc11", size = 35577989, upload-time = "2026-06-19T15:00:02.235Z" }, + { url = "https://files.pythonhosted.org/packages/84/ca/210d4759c7210bb7d269437421959b39a33434e2776b60c5cb8a763bb30a/scipy-1.18.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4a55985d54c769c872e64b7f4c8a81cc30ef700cc04296abbbf3705439c126de", size = 37421717, upload-time = "2026-06-19T15:00:05.102Z" }, + { url = "https://files.pythonhosted.org/packages/2b/54/9a9edb45345bd6744da5ddfb6628e5d5185920494c6a67ec45b6381004cb/scipy-1.18.0-cp312-cp312-win_amd64.whl", hash = "sha256:71ccc8faa2dd16ac310233203474a8b5cb67f10dedd54a3116d34943f4b19132", size = 36597428, upload-time = "2026-06-19T15:00:08.112Z" }, + { url = "https://files.pythonhosted.org/packages/99/0e/33f32a2a58987e26aec0f7df252cbbad1e90ae77bdbc76f40dd4ed0cf0ea/scipy-1.18.0-cp312-cp312-win_arm64.whl", hash = "sha256:d88363fd9d8fbd3511bd273f1a49efb2a540773ddf92a91d57498ce7dd7f3e76", size = 24351481, upload-time = "2026-06-19T15:00:11.103Z" }, + { url = "https://files.pythonhosted.org/packages/05/52/9c0136c2de7ae0779b7b366447766cec6d9f0702c56bb8ffeb04c8fd3af4/scipy-1.18.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:09143f676d157d9f546d663504ef9c1becb819824f1afc018814176411942446", size = 31036107, upload-time = "2026-06-19T15:00:14.03Z" }, + { url = "https://files.pythonhosted.org/packages/02/73/0291a64843270f4efb86cdcf2ee0f2048631b65ec6b405398b2b4dbf11bf/scipy-1.18.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5efe260f69417b97ddae455bfb5a95e8359f7f66ad7fa9522a60feb66f169520", size = 28663303, upload-time = "2026-06-19T15:00:16.819Z" }, + { url = "https://files.pythonhosted.org/packages/d3/0f/10ffa0b697a572f4e0d48b92a88895d366422f019f723e7e14a84c050dac/scipy-1.18.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:68363b7eaacd8b5dd426df56d782cc156468ac79a127a1b87ca597d6e2e82197", size = 20404960, upload-time = "2026-06-19T15:00:19.635Z" }, + { url = "https://files.pythonhosted.org/packages/7e/d2/e896cea21ba8edd6c81d4c55b1ffcc717e79698dcbebf9641b4cfb4c6622/scipy-1.18.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:c5557d8be5da8e41353fcd4d21491fdbab83b062fc579e94dc09a7c8ab4f669b", size = 23034074, upload-time = "2026-06-19T15:00:22.107Z" }, + { url = "https://files.pythonhosted.org/packages/ea/b2/e83ea34279a52c03374477c74006256ec78df65fc877baa4617d6de1d202/scipy-1.18.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0d13bca67c096d89fb95ced0d8921807300fce0275643aef9533cc63a0773468", size = 33942038, upload-time = "2026-06-19T15:00:24.964Z" }, + { url = "https://files.pythonhosted.org/packages/f6/af/e8fe5fb136f51e2b01678b92cb4106d10d8cd68ec147ead2e7cb0ac75398/scipy-1.18.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a46f9273dbd0eb1cefba61c9b8648b4dfe3cbc14a080176f9a73e44b8336dc7f", size = 35266390, upload-time = "2026-06-19T15:00:28.059Z" }, + { url = "https://files.pythonhosted.org/packages/3a/49/2c5cbb907b56695fc67517811d1db234dfd83381a84814ec220aded2794d/scipy-1.18.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5aba46108853ddfc77906b6557aac839d2b52e900c1d72a1180adaaab58d265f", size = 35551324, upload-time = "2026-06-19T15:00:31.014Z" }, + { url = "https://files.pythonhosted.org/packages/bb/73/eda39f7a2d306ff0ffc574afd13c0bbb6d10a603d9a413998ee269487a80/scipy-1.18.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b6f758e35f12757b5d95c00bc6de2438e229c2664b7a92e96f205959d9f2dfa4", size = 37404785, upload-time = "2026-06-19T15:00:34.072Z" }, + { url = "https://files.pythonhosted.org/packages/b7/d2/ae881ee28d014f38e0ccbfd974a06a919ba9af34f1f74bf42b5301891d63/scipy-1.18.0-cp313-cp313-win_amd64.whl", hash = "sha256:1afac4a847207c7ff8efd321734a50b06d0280b3b2a2c0fc2f413101747ad7c7", size = 36554943, upload-time = "2026-06-19T15:00:36.903Z" }, + { url = "https://files.pythonhosted.org/packages/70/3a/21154e2d54eb3639c6bf4dbae2e531c68356bfe95990daa30df33b30d556/scipy-1.18.0-cp313-cp313-win_arm64.whl", hash = "sha256:c5dbddf60e58c2312316d097271a8e73d40eaf2eabfa4d95ed7d3695bbf2ce7b", size = 24350911, upload-time = "2026-06-19T15:00:40.062Z" }, + { url = "https://files.pythonhosted.org/packages/78/b5/915a19b3de2f7430062b509653563db1633ddbb6f021b06731521115d4e2/scipy-1.18.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4c256ee70c0d1a8a2ace807e199ccd4e3f57037433842abb3fb36bc17eaa9578", size = 31036253, upload-time = "2026-06-19T15:00:43.216Z" }, + { url = "https://files.pythonhosted.org/packages/d7/88/b72def7262e150d16be13fca37a96481138d624e700340bc3362a7588929/scipy-1.18.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:2ef3abc54a4ffc53765374b0d5728532dfdd2585ed23f6b11c206a1f0b1b9af8", size = 28673758, upload-time = "2026-06-19T15:00:46.663Z" }, + { url = "https://files.pythonhosted.org/packages/91/02/2e636a61a525632c373cf6a9c24442a3ffb79e364d38e98b32042964ac32/scipy-1.18.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:f2a6af57bd9e4a75d70e4117e78a1bbee84f79ae3fbb6d0111005d6ebcc4cb8d", size = 20415514, upload-time = "2026-06-19T15:00:49.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b6/2135974442f6aba159d9d39d774a1c8cb19947016725d69fecc685df45bf/scipy-1.18.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:3f1ac564d3bf6c03d861d2cd87a1bea0da2887136f7fb1bf519c05a8971452d6", size = 23034398, upload-time = "2026-06-19T15:00:51.941Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e6/ba89ec5abf6ee9257c0d1ec985573f3ae32742c24bc03e016388a40b1b15/scipy-1.18.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40395a5fcd1abee49a5c7aaa98c29db393eedc835138560a588c47ec16156690", size = 33998032, upload-time = "2026-06-19T15:00:54.838Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c4/bc41eb19b0fd0db868f4132920879019318d80cc522ad8f2bca4611af808/scipy-1.18.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ca01e8ae69f1b18e9a58d91afead31be3cef0dd905a10249dac559ee15460a0", size = 35283333, upload-time = "2026-06-19T15:00:58.152Z" }, + { url = "https://files.pythonhosted.org/packages/53/a4/cbdeef6eb3830a8462a9d4ada814de5fc984345cc9ecf17cbec51a036f1e/scipy-1.18.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7a7f3b01647384dbc3a711e8c6778e0aabbe93959249fef5c7393396bcac0867", size = 35610216, upload-time = "2026-06-19T15:01:01.155Z" }, + { url = "https://files.pythonhosted.org/packages/80/4d/b2b82502b65f661d1b789c1665dcdf315d5f12194e06fc0b37946294ebae/scipy-1.18.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6aa94e78ec192a30063a5e72e561c28af769dc311190b24fe91774eff1969709", size = 37418960, upload-time = "2026-06-19T15:01:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/93/3e/902d836831474b0ab5a37d16404f7bc5fafd9efba632890e271ba952635f/scipy-1.18.0-cp314-cp314-win_amd64.whl", hash = "sha256:2d8bbdc6c817f5b4006a54d799d4f5bab6f910193cbb9a1ff310833d4d270f61", size = 37288845, upload-time = "2026-06-19T15:01:07.822Z" }, + { url = "https://files.pythonhosted.org/packages/b6/43/8d73b337a3bdb14daa0314f0434210747c02d79d729ce1777574a817dcf6/scipy-1.18.0-cp314-cp314-win_arm64.whl", hash = "sha256:18e9575f1569b2c54174e6159d32942e03731177f63dce7975f0a0c88d102f5b", size = 24988971, upload-time = "2026-06-19T15:01:11.076Z" }, + { url = "https://files.pythonhosted.org/packages/b4/b4/f11918b0508a2787031a0499a03fbe3546f3bb5ca05d01038c45b278c09a/scipy-1.18.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:f351e0dd702687d12a402b867a1b4146a256923e1c38317cbc472f6372b94707", size = 31399325, upload-time = "2026-06-19T15:01:13.723Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d1/1f287b57c0ff0ee5185dff3946d92c8017d39b0e431f0ae79a3ff1859512/scipy-1.18.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:7c7a51b33ce387193c97f228320cf8e87361daa1bba750638677729598b3e677", size = 29092110, upload-time = "2026-06-19T15:01:16.908Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1a/7b74eb6c392fdcb27d414c0e7558a6d0231eb3b6d73571f479bb81ea8794/scipy-1.18.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:84031d7b052a54fae2f8632e0ec802073d385476eb9a63079bce6e23ef9283d4", size = 20833811, upload-time = "2026-06-19T15:01:20.488Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ad/f3941716320a7b9cb4d68734a903b45fe16eff5fb7da7e16f2e619304979/scipy-1.18.0-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:56abf29a7c067dde59be8b9a22d606a4ea1b2f2a4b756d9d903c62818f5dacce", size = 23396644, upload-time = "2026-06-19T15:01:23.364Z" }, + { url = "https://files.pythonhosted.org/packages/22/22/1446b62ffe07f9719b7d9b1b6a4e05a772833ae8f441fe4c22c34c9b250f/scipy-1.18.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ad44305cfa24b1ba5803cbbebf033590ccbac1aa5d612d727b785325ab408b0", size = 34079318, upload-time = "2026-06-19T15:01:26.002Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/b87da667098bb470fa30c7011b0ba351ee976dd395c78798c66e941665a3/scipy-1.18.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:945c1761b93f38d7f99ae81ae80c63e621471608c7eeead563f6df025585cd58", size = 35324320, upload-time = "2026-06-19T15:01:28.881Z" }, + { url = "https://files.pythonhosted.org/packages/f8/a1/c7932f91909759b0267f75fdea34e91309f96b895757534b76a90b6b4344/scipy-1.18.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1a4441f15d620578772a49e5ab48c0ee1f7a0220e387110283062729136b2553", size = 35699541, upload-time = "2026-06-19T15:01:31.968Z" }, + { url = "https://files.pythonhosted.org/packages/f7/86/5185061a1fcc41d18c5dc2463969b3a3964b31d9ac67b2fb05d4c7ff7670/scipy-1.18.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9aac6192fac56bf2ca534389d24623f07b39ff83317d58287285e7fbd622ff76", size = 37472480, upload-time = "2026-06-19T15:01:35.136Z" }, + { url = "https://files.pythonhosted.org/packages/31/8e/f04c68e39919a010d34f2ee1367fd705b0a25a02f609d755f0bfbc0a15fc/scipy-1.18.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e40baea28ae7f5475c779741e2d90b1247c78531207b49c7030e698ff81cee3f", size = 37365390, upload-time = "2026-06-19T15:01:38.091Z" }, + { url = "https://files.pythonhosted.org/packages/d5/19/969dc072906c84dd0a3b05dcf57ea750936087d7873549e408b35cfc3f97/scipy-1.18.0-cp314-cp314t-win_arm64.whl", hash = "sha256:368e0a705903c466aa5f08eefb39e6b1b6b2d659e7352a31fd9e2438365be0f8", size = 25279661, upload-time = "2026-06-19T15:01:40.817Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "smart-open" +version = "8.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/53/9c513747547fd595d5c143259129ea8b9c3ea2f6b7bb9dcea2b1966ded3c/smart_open-8.0.1.tar.gz", hash = "sha256:18b1c4496003c6902be17c15f032b5c319f307c89c6ae9e6b028b508bed8b2cf", size = 61882, upload-time = "2026-07-15T13:56:10.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/96/325b8c507ccecc50421fecc0345a502ee6e4a44785af3c4e6ecbadad624a/smart_open-8.0.1-py3-none-any.whl", hash = "sha256:3e97f90e92a952cb57863dfe132082c400a52eeeb27c067692fb51dbcc5b0089", size = 73504, upload-time = "2026-07-15T13:56:09.033Z" }, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + +[[package]] +name = "soupsieve" +version = "2.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/38/e12680bbe6b4f8f3d17adcaf38d26850aa756c85cf4a80e79fc12a018fe8/soupsieve-2.9.1.tar.gz", hash = "sha256:c33e6605bbc71dd628b00c632d58ae607c22bade247e52553928f83bbb75b4ba", size = 122261, upload-time = "2026-07-21T16:57:17.452Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/2c/437fe806897c2d6cfdc3ee43a18da8bf8e568530a4ae9bac781541ca9896/soupsieve-2.9.1-py3-none-any.whl", hash = "sha256:4f4477399246b7a0c720a88ca2454b11cd6bb9ae4c9d170140786e916776c14c", size = 37404, upload-time = "2026-07-21T16:57:16.421Z" }, +] + +[[package]] +name = "stack-data" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asttokens" }, + { name = "executing" }, + { name = "pure-eval" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707, upload-time = "2023-09-30T13:58:05.479Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, +] + +[[package]] +name = "tabulate" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/58/8c37dea7bbf769b20d58e7ace7e5edfe65b849442b00ffcdd56be88697c6/tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d", size = 91754, upload-time = "2026-03-04T18:55:34.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, +] + +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, +] + +[[package]] +name = "tifffile" +version = "2026.7.31" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/62/083288b8d6b9ecb2968e7573e60aa0694089e960e203934bc217169acf64/tifffile-2026.7.31.tar.gz", hash = "sha256:79b1f4b1aba3ef3e6b6f1691a32abb62f5d7383faa52a12771c695232ba40bee", size = 442177, upload-time = "2026-08-01T02:28:32.523Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/ed/75bf4d6ae6fec7233ef466f27dffc99f91fde53a31e69f02640b418317ec/tifffile-2026.7.31-py3-none-any.whl", hash = "sha256:81adfa08012be1c478f99b83cda2f529eef8620cfbdf94fc41eef6f1d7b47dc5", size = 271576, upload-time = "2026-08-01T02:28:31.068Z" }, +] + +[[package]] +name = "tiktoken" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "regex" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/e5/5f3cb2159769d0f4324c0e9e87f9de3c4b1cd45848a96b2eb3566ad5ca77/tiktoken-0.13.0.tar.gz", hash = "sha256:c9435714c3a84c2319499de9a300c0e604449dd0799ff246458b3bb6a7f433c1", size = 38986, upload-time = "2026-05-15T04:51:27.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/8e/144bde4e01df66b34bb865557c7cd754ed08b036217ebd79c9db5e9048a9/tiktoken-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:32ac870a806cfb260a02d0cb70426aef02e038297f8ad50df5040bb5af360791", size = 1034888, upload-time = "2026-05-15T04:50:31.579Z" }, + { url = "https://files.pythonhosted.org/packages/36/18/d4ac9d20956cdebca04841316660ed584c2fecdc2b81722a28bc7ad3b1e4/tiktoken-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4d9980f11429ed2d737c463bb1fb78cf330caa026adf002f714aced7849a687b", size = 982970, upload-time = "2026-05-15T04:50:32.961Z" }, + { url = "https://files.pythonhosted.org/packages/74/ed/6bb8d05b9f731f749fee5c6f5ca63e981143c826a5985877330507bd13b7/tiktoken-0.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3f277ebea5edd7b8bf03c6f9431e1d67d517530115572b2dc1d465326e8f88c7", size = 1115741, upload-time = "2026-05-15T04:50:34.475Z" }, + { url = "https://files.pythonhosted.org/packages/34/de/2ca96b07a82d972b74fe4b46de055b79c904e45c7eab699354a0bfa697dc/tiktoken-0.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a116178fa7e1b4065bff05214360373a65cac22f965be7b3f73d00a0dbfe7649", size = 1136523, upload-time = "2026-05-15T04:50:35.782Z" }, + { url = "https://files.pythonhosted.org/packages/ee/dc/9dafec002c2d4424378563cf4cf5c7fb93631d2a55013c8b87554ee4012c/tiktoken-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c397ddda233208345b01bd30f2fca79ff730e55731d0108a603f9bc57f6af3b", size = 1181954, upload-time = "2026-05-15T04:50:36.99Z" }, + { url = "https://files.pythonhosted.org/packages/a1/d0/1f8578c45b2f24759b46f0b50d31878c63c73e6bf0f2227e10ec5c5408dc/tiktoken-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:95097e4f89b06403976e498abf61a0ee73a7497e73fb599cb211d8197a054d91", size = 1240069, upload-time = "2026-05-15T04:50:38.221Z" }, + { url = "https://files.pythonhosted.org/packages/aa/90/28d7f154888610aa9237e541986beb62b479df29d193a5a0617dbb1514d0/tiktoken-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f2d16e7a7c783ad81f36e457d046d1f1c8af70b22aec8a13238efe531977c41", size = 874748, upload-time = "2026-05-15T04:50:39.587Z" }, + { url = "https://files.pythonhosted.org/packages/9c/83/b096c859c2a47c11731bf2f5885f4028b809dfe2396582883eed9cae372f/tiktoken-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5df5d1507bd245f1ccad4a074698240021239e455eb0bb4ced4e3d7181872154", size = 1034228, upload-time = "2026-05-15T04:50:40.988Z" }, + { url = "https://files.pythonhosted.org/packages/53/61/c68e123b6d753e3fc2751e9b18e732c9d8bf1e1926762e736eee935d931c/tiktoken-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fe806a50664e83a6ffd56cbd1e4f5dcc6cd32a3e7538f70dc38b1a271384545", size = 982978, upload-time = "2026-05-15T04:50:42.195Z" }, + { url = "https://files.pythonhosted.org/packages/ef/8b/96cc178cc584e65d363134500f297790b06cd48cdeb1e8fcf7bbe60f4715/tiktoken-0.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:125bc05005e747f993a83dc67934249932d6e4209854452cd4c0b1d53fba3ba2", size = 1116355, upload-time = "2026-05-15T04:50:43.564Z" }, + { url = "https://files.pythonhosted.org/packages/86/f5/bab735d2c72ea55404b295d02d092644eb5f7cc6205e34d35eb9abfb9ab2/tiktoken-0.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5e6358911cab4adee6712da27d65573496a4f68cf8a2b5fca6a4ad10fc5748cf", size = 1135772, upload-time = "2026-05-15T04:50:44.782Z" }, + { url = "https://files.pythonhosted.org/packages/4e/b9/6de04ebdf904edfaad87788011b3735087a0c9ea671b9027e1e4e965e8c8/tiktoken-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:975cbd78d085d75d26b59660e262736dcaed1e35f8f142cd6291025c01d25486", size = 1182415, upload-time = "2026-05-15T04:50:46.422Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9c/470a05f3b1caf038f44880e334d47ab674e0c80d514c66b375d14d5afa10/tiktoken-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75ab9bc99fa020a4c283424590ecd7f3afd70c1c281cb3fa3192a6c3af9f9615", size = 1239879, upload-time = "2026-05-15T04:50:48.052Z" }, + { url = "https://files.pythonhosted.org/packages/42/a6/c1936d16055436cb32e6c6128d68629622e00f4768562f55653752d34768/tiktoken-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:6b1615f0ff71953d19729ceb18865429c185b0a23c5353f1bbca34a394bf60f7", size = 874829, upload-time = "2026-05-15T04:50:49.202Z" }, + { url = "https://files.pythonhosted.org/packages/d6/07/acb5992c3772b5a36284f742cfb7a5895aa4471d1848ac31464ad50d7fdf/tiktoken-0.13.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6eb4a5bfbc6426938026b1a334e898ac53541360d62d8c689870160cc80abd67", size = 1033600, upload-time = "2026-05-15T04:50:50.4Z" }, + { url = "https://files.pythonhosted.org/packages/14/e9/742e9aec30f59b9f161f7ff7cd072e02ea836c9e1c0854a8076dfcd40d5c/tiktoken-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:43cee3e5400573b2046fbf092cc7a5bc30164f9e4c95ce20714da929df48737a", size = 982516, upload-time = "2026-05-15T04:50:52.03Z" }, + { url = "https://files.pythonhosted.org/packages/72/74/ca1541b053e7648254d2e4b42a253e1bb4359f2c91a0a8d49228c794e1a0/tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7de52e3f566d19b3b11bd37eea552c6c305ad74081f736882bd44d148ed4c48d", size = 1115518, upload-time = "2026-05-15T04:50:53.543Z" }, + { url = "https://files.pythonhosted.org/packages/46/e3/93825eaf5a4a504795b787e5d5dea07fbeb3dabf97aa7b450be8bde59c89/tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:51384448aa508e4df84c0f7c1dc3211c7f7b8096325660ee5fc82f3e11b381ce", size = 1136867, upload-time = "2026-05-15T04:50:55.191Z" }, + { url = "https://files.pythonhosted.org/packages/8c/46/002b68de6827091d5ae90b048f326e8aad8d953520950e5ce1508879414f/tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e28157350f7ebf35008dd8e9e0fdb621f976e4230c881099c85e8cf07eaa50e2", size = 1181826, upload-time = "2026-05-15T04:50:56.296Z" }, + { url = "https://files.pythonhosted.org/packages/db/c6/d393e3185a276505182f7abd93fe714f3c444a2be9180798fa052347504e/tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:165cf1820ea4a354985c2490a5205d4cc74661c934aca79dd0368232fff94e0f", size = 1239489, upload-time = "2026-05-15T04:50:57.918Z" }, + { url = "https://files.pythonhosted.org/packages/b7/4d/bc07d1f1635d4897a202acc0ae11c2886eaa7325c359ba4741b47bf8e225/tiktoken-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6c43a675ca14f6f2749ba7f12075d37456015a24b859f2517b9beb4ef30807ec", size = 873820, upload-time = "2026-05-15T04:50:59.528Z" }, + { url = "https://files.pythonhosted.org/packages/8c/93/0dd6adca026a616c3a92974566b43381eea4b475ce1f36c062b8271a9ac5/tiktoken-0.13.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaaaef47c2406277181d2086484c317bf7fc433e2d5d03ff94f56b0dcec87471", size = 1034977, upload-time = "2026-05-15T04:51:00.957Z" }, + { url = "https://files.pythonhosted.org/packages/d9/77/5ec6e6bc5b30bed6d93f7f2162d8f6b32437b3ba27cb527cfe004f6109c9/tiktoken-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ca8b310bd93b3772cb1b7922d915446864860f562bdfe4825c63a0aed3fb28cd", size = 983635, upload-time = "2026-05-15T04:51:02.629Z" }, + { url = "https://files.pythonhosted.org/packages/94/b0/c8ae9aff00d625c50659b4513e707a0462c4bf5d4d6cc1b802103225c02e/tiktoken-0.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:32e0c12305105002c047b3bb1070b0dd9a73b0cb3b2856a8972b810e7a4f5881", size = 1116036, upload-time = "2026-05-15T04:51:04.082Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ac/6a5dddd1d0a6018ecb389bd0353e6b4a515eb4d2286611bd0ace1937b9e1/tiktoken-0.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:5ba5fd62507a932d1241346179e3b39bc7bf7408f03c272652d93b3bedf5db24", size = 1135544, upload-time = "2026-05-15T04:51:05.229Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b8/585032b4384b2f7dcdaddcb52865c83a701a420d09e3c2b4a2be1c450c57/tiktoken-0.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d108bc2d470fc53c8ecd24f2c0fd2b5f98c33e87cdb6aa2e9b8c5dced703d273", size = 1182217, upload-time = "2026-05-15T04:51:06.517Z" }, + { url = "https://files.pythonhosted.org/packages/cd/b6/993ff1ded3958215fd341a847b8e5ffeb5de473f435296870d314fc91ac4/tiktoken-0.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cb99cb5127449f58d0a2d5f5ccfb390d8dbdfd919c221246caaee29d8725ed51", size = 1239404, upload-time = "2026-05-15T04:51:07.843Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3d/fef7e06e3b33e7538db0ced734cf9fe23b6832d2ac4990c119c377aec55e/tiktoken-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:115c4f26ffa11caac8b54eea35c2ad38c612c20a48d35dd15d70a02ac6f51f58", size = 918686, upload-time = "2026-05-15T04:51:08.925Z" }, + { url = "https://files.pythonhosted.org/packages/c1/82/a7fc44582bc32ab00de988a2299bf77c077f59068b233109e34b7d6ca7e6/tiktoken-0.13.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:472527e9132952f2fbf77cd290658bacf003d4d5a3fabc18e5fbd407cbae4d9b", size = 1034454, upload-time = "2026-05-15T04:51:10.035Z" }, + { url = "https://files.pythonhosted.org/packages/37/d0/24d8a890c14f432a05cea669c17bebeaa99f96a7c79523b590f564246411/tiktoken-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4e2f67d27c9626cdd25fe33d9313c5cdb3d8d82da646b68d6eb8e7e9c20e6448", size = 982976, upload-time = "2026-05-15T04:51:11.23Z" }, + { url = "https://files.pythonhosted.org/packages/49/b7/2ab43f62788a9266187a9bfc1d3af99ad83e5eaa25fbef168a69cd5ad14f/tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:2b920b35805cd64585a37c3dc7ce65fba4d2d36016be01e1d7942482ca29093a", size = 1115526, upload-time = "2026-05-15T04:51:12.608Z" }, + { url = "https://files.pythonhosted.org/packages/64/39/1494321ed323ce7a14d88e3cd6cb9058625977df1c6961ddc492bd10a9f3/tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:493af3aa28a4aaf2e3d2600a2ee717252c9bf5ab38fff94eb5a02db5ab77e5ad", size = 1136466, upload-time = "2026-05-15T04:51:13.926Z" }, + { url = "https://files.pythonhosted.org/packages/96/d9/dfd086aa2d918c563a140720e0ce296cada1634efd2783d5cf51e05f984e/tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6644c9c2b5cf3916f5a3641d7d12fdb3f006a7b3d9ff6acdaec44e29ab1ff91e", size = 1181863, upload-time = "2026-05-15T04:51:15.025Z" }, + { url = "https://files.pythonhosted.org/packages/2f/68/a18b4f307086954fdae32714cb4f85562e34f9d34ab206e61f1816aa6018/tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cb65b60b9408563676d874a3a4ee573370066f0dc4e29d84e82e989c6517424", size = 1239218, upload-time = "2026-05-15T04:51:16.103Z" }, + { url = "https://files.pythonhosted.org/packages/16/5b/f2aa703a4fc5d2dff73460a7d46cc2f3f44aa0f3dd8eeb20d2a0ecf68862/tiktoken-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:85b78cc3a2c3d48723ca751fa981f1fedccd54194ca0471b957364353a898b07", size = 918110, upload-time = "2026-05-15T04:51:17.237Z" }, +] + +[[package]] +name = "tqdm" +version = "4.70.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/3b/6c24bec5be5e743ffd99576daa5cc077722fc7d5bbc00bd133fa0c698dc6/tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220", size = 795438, upload-time = "2026-07-27T11:33:15.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953", size = 80184, upload-time = "2026-07-27T11:33:13.167Z" }, +] + +[[package]] +name = "traitlets" +version = "5.16.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/2e/a7fbfe268c8a3b32546930c0297c101d65a4a14c304ad5790a9f478f0e4e/traitlets-5.16.1.tar.gz", hash = "sha256:ed900c2b631aa3a112811139fa97b8d2c3bad5e989656bba4b7e52c7852c18c1", size = 166137, upload-time = "2026-08-03T08:32:36.848Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/66/0d785f0bc5e4315a96c989bb476d0fc07ea4f85132550c7b156ca2035d52/traitlets-5.16.1-py3-none-any.whl", hash = "sha256:f775618166caa0396c8e337099240f2bd3e5e917d203b2e6fbe21a58d3cb1f6b", size = 86211, upload-time = "2026-08-03T08:32:34.48Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[[package]] +name = "uuid-utils" +version = "0.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/91/63938e0e7e7876658e5e40178e7c0735b53527886fe11797a11699c55edd/uuid_utils-0.17.0.tar.gz", hash = "sha256:abb5667a36119019b3fa320c4d10c21ebccfcc87c8a739e6a0056cee7f48dde2", size = 43220, upload-time = "2026-07-09T13:49:58.433Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/80/a7e685968e3cec99d6fe2fb25d0f5726310e1bba356da68c13dfd8b7d140/uuid_utils-0.17.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:9205068badf453d2f0821fd5d340389b4679992d7ff79d4f3e5608996dd1b287", size = 556403, upload-time = "2026-07-09T13:48:27.022Z" }, + { url = "https://files.pythonhosted.org/packages/56/47/3102d93bcb7b0bfe6bede63ff8f221a7f91348e10a37f682773be27c56d9/uuid_utils-0.17.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0fcca4e838af9ac9243b3358d7c14afa4dca286a87781124c272d6c4cad9c968", size = 285608, upload-time = "2026-07-09T13:48:28.769Z" }, + { url = "https://files.pythonhosted.org/packages/55/fb/d59695f0f8db065b93c63316eaafa05a22d75a0486978a33736c52c646d5/uuid_utils-0.17.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f3729e839209f3457d0d8b6a35a376fdf65577a5aecaf4cc3587d3305759ba6", size = 319926, upload-time = "2026-07-09T13:48:29.965Z" }, + { url = "https://files.pythonhosted.org/packages/5a/03/62fabcd1e990e07a0e220e8d552af45bc16f107fa8e55c2014a706bb1a1e/uuid_utils-0.17.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3dac0ad0cd9a2818d1775215365a4e8c2f8ada215529dd26f3f8cceeb67a6988", size = 327172, upload-time = "2026-07-09T13:48:31.187Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/a5081391338b459e2f8d8b12581f00f8caa6317fab510e0e85c18c59e938/uuid_utils-0.17.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e671b2322ef09106ecb1ca0f4c398b134d5e2c1f80d7a4f3336847a3072c0e94", size = 439075, upload-time = "2026-07-09T13:48:32.295Z" }, + { url = "https://files.pythonhosted.org/packages/59/30/91795bd01e17a13661280d4899fbf38fb05e3f38e873f9aaec106ec30aa0/uuid_utils-0.17.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8eb3e5caca8d3a6f72ea4cce024583f989f6f2e9186f98800213fff0176e8bcc", size = 320247, upload-time = "2026-07-09T13:48:33.64Z" }, + { url = "https://files.pythonhosted.org/packages/e5/11/09102b78303e4eb62069d6d88ef9fd661dc523e8f429e1fd67eaa78a6f44/uuid_utils-0.17.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8b72c2002202038666bf647f9a790906214c7c11cd0d6efef77b7d07bef3034a", size = 344738, upload-time = "2026-07-09T13:48:34.786Z" }, + { url = "https://files.pythonhosted.org/packages/74/f9/be95bad6954b60328878c3800258f01a6accd24fd75112d13f023462d53f/uuid_utils-0.17.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4e2ac1c0b56f2c91b6f158e29ed96b1503223fe8aa6e79b1be1dc55bd8a5131c", size = 496845, upload-time = "2026-07-09T13:48:36.057Z" }, + { url = "https://files.pythonhosted.org/packages/2d/02/8a19a34e0530d987488a068a71576a236f5c8c746630b870b57f71eb24ef/uuid_utils-0.17.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6c142bd0cb4dba31c10babe00d59f7ef6460f0ef55eaa9c1a9da270684af996a", size = 603233, upload-time = "2026-07-09T13:48:37.512Z" }, + { url = "https://files.pythonhosted.org/packages/f4/a8/b1abab36ff73b0248d82179816467f6d39a2e80fd64329a895ca94f3508e/uuid_utils-0.17.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e252db239eb41c32248e096e0d170bce5896a4fd3405556362bc3dd83d912206", size = 561401, upload-time = "2026-07-09T13:48:38.977Z" }, + { url = "https://files.pythonhosted.org/packages/61/91/70e7b528b351cc03a9ca43e6116371cdde31bb12bcead7ca2ca1367366cc/uuid_utils-0.17.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:237722b6581bb5b4eb4cefbcbe5c6e2980a440aabe781fbe50ebf1cb71eee4cc", size = 525314, upload-time = "2026-07-09T13:48:40.599Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f6/9167e90cf9937d6558f92d022ff3024a69d938a514d9c8faa4080f73b001/uuid_utils-0.17.0-cp312-cp312-win32.whl", hash = "sha256:46a73cacdf512f473a81f65dbf84186e08cfe6e9118fa582b6c6b33a8288a30d", size = 166831, upload-time = "2026-07-09T13:48:41.862Z" }, + { url = "https://files.pythonhosted.org/packages/5c/7d/0b889654d9ee3413f810cf4685e241285f650d98a4103ac9f3c6bcc95f29/uuid_utils-0.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:e59b60a0a4cb7541480e02090d37dc2df3b72df4c2e776fff64ce3a4e3dd4637", size = 172944, upload-time = "2026-07-09T13:48:42.992Z" }, + { url = "https://files.pythonhosted.org/packages/be/35/8c6e1bf65e4d400352885dadc656ad6d0af96e89231e3f04686bc2197128/uuid_utils-0.17.0-cp312-cp312-win_arm64.whl", hash = "sha256:d561a4c5747a1e6c7fa7c49a0292e78b4e8c456332caa084fc7abad8de828652", size = 172459, upload-time = "2026-07-09T13:48:44.271Z" }, + { url = "https://files.pythonhosted.org/packages/d2/dd/614fb9912157ac0128e6050859ccf06d9f13df9a944a803e8f80f6157e38/uuid_utils-0.17.0-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:d11a7bc1e02da8984d32e6de9e0826c6edac00eac17de270f372bf32f9a0af63", size = 557259, upload-time = "2026-07-09T13:48:45.664Z" }, + { url = "https://files.pythonhosted.org/packages/3e/11/d072711704de3d21bec08b6c2f36a215200ca1d5e01a390ea1ac434080a0/uuid_utils-0.17.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7a49f47ac26df3e431c56b825c1bae8e6d3d591fdbb7438c227cc9845a7e3d73", size = 286271, upload-time = "2026-07-09T13:48:47.018Z" }, + { url = "https://files.pythonhosted.org/packages/18/6d/8a63e5eb2d5a6ba69a6c2036e305075bd6f5a022e7ea25fc6ce0eb7c51d2/uuid_utils-0.17.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32df1944808877702ceea398c103881c09a679bb672a215e01c2a84231266bf9", size = 320025, upload-time = "2026-07-09T13:48:48.208Z" }, + { url = "https://files.pythonhosted.org/packages/f7/2d/bdc2caf9719d9090d7c46043242ae6136cba4f7a7ee384992ab905ad9aa1/uuid_utils-0.17.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:98c88d3edd08e7245562e9815996dbc6f0bd4745e1c76462f24af5ae4e187dd1", size = 327931, upload-time = "2026-07-09T13:48:49.673Z" }, + { url = "https://files.pythonhosted.org/packages/b6/33/9219d09d51ead282b578b2a4e0a515c2cce3ec52076cada8bfb7e35727d5/uuid_utils-0.17.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a4370089c8b2e42f1db51d76408c7fa8eaa2934bf854d17983d16179c07c098", size = 438537, upload-time = "2026-07-09T13:48:50.842Z" }, + { url = "https://files.pythonhosted.org/packages/d8/79/e8e0f8b3955f2081c116157119d87659937893242eb834aa170da04d660b/uuid_utils-0.17.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:09a55b7a5ae764985cb46467496a1787678d0a1400356157a080ad95b1a36869", size = 320656, upload-time = "2026-07-09T13:48:52.164Z" }, + { url = "https://files.pythonhosted.org/packages/d5/5e/d1ceddc430ff04b6e21704b2030d4438074a2f478b265dab43da957791c1/uuid_utils-0.17.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:56aa6488b931246fae11924e4bd0e2b32677e63945eecb71c29e3c2ca0dc3131", size = 345310, upload-time = "2026-07-09T13:48:54.076Z" }, + { url = "https://files.pythonhosted.org/packages/d5/62/89438e12f389a843e626b7e37691319a057b3d6b80914609106891faadda/uuid_utils-0.17.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:309a35f12d99dde19032bc2259cda6431c85eeac0879134dc777cc3087d7e1cb", size = 496771, upload-time = "2026-07-09T13:48:55.365Z" }, + { url = "https://files.pythonhosted.org/packages/87/d2/eedcd99f522d60e238ead03844f0d51743ba84d33044959e230b756bf212/uuid_utils-0.17.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:21c79b61ff750abcf057163dd764ccb6196cde7a26cda1b31b45cd97769e03b3", size = 603631, upload-time = "2026-07-09T13:48:56.746Z" }, + { url = "https://files.pythonhosted.org/packages/0e/a8/bb1b38aaddd7243b6e562c6694f499bf094800918316192fd8cb2cdc2620/uuid_utils-0.17.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4134353bfe3026ddab8e886002dc52bc5a0ab04611aabb0eaae23c32e6e57f64", size = 562008, upload-time = "2026-07-09T13:48:58.241Z" }, + { url = "https://files.pythonhosted.org/packages/b4/77/5f7ed930dc105e293845c09e4d5bd84076318a12f45a46783e1af64906d7/uuid_utils-0.17.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7c89359affecebe2e39e6a116d069b363c936511a9572b308402489a26957d89", size = 525527, upload-time = "2026-07-09T13:48:59.784Z" }, + { url = "https://files.pythonhosted.org/packages/fd/25/1b55697adf6811a6f92cff6340e6b03e31fd6bc51066a5c10698c29b3679/uuid_utils-0.17.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:6a019a31bc4db89a0903a3e4f6b218571f3a6ff0ad4b3d3fe1c8f91a05ff6e3e", size = 97965, upload-time = "2026-07-09T13:49:01.217Z" }, + { url = "https://files.pythonhosted.org/packages/26/bf/cd729343de4684230be8a966bad7bfc2cf10ce3e643b1189a8b5370dbe35/uuid_utils-0.17.0-cp313-cp313-win32.whl", hash = "sha256:b3131a82d0c7611f0aa480a6d36929e001a3f54ba0fc029a8118a5863cce513c", size = 167316, upload-time = "2026-07-09T13:49:02.354Z" }, + { url = "https://files.pythonhosted.org/packages/76/f0/e602ae0a1b139a7826e5189b93d91902564def06d5006324fd2faf82c8fc/uuid_utils-0.17.0-cp313-cp313-win_amd64.whl", hash = "sha256:9e311f908d2f842fca4c7dcebc4f10306b8089b204ef04cf6704b4332c9ff6ff", size = 173630, upload-time = "2026-07-09T13:49:03.529Z" }, + { url = "https://files.pythonhosted.org/packages/1a/52/024ebece265b387154115dc4f1d9727174ef82623069f4bec8b7ed7e73f7/uuid_utils-0.17.0-cp313-cp313-win_arm64.whl", hash = "sha256:c351737e2e65497c7200ab4ffb8af97e9f48be6488309abdd265fe08d66ee92f", size = 173214, upload-time = "2026-07-09T13:49:04.836Z" }, + { url = "https://files.pythonhosted.org/packages/56/44/e2fd3fdf356e1b55d2acf1b956b4f3f29ffb215a99c387eba04b1c5fba66/uuid_utils-0.17.0-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:673d89cc434cc9b97a0b4cf61272f6fca70a81f64eb0afbface2a0d9f77f06cd", size = 562232, upload-time = "2026-07-09T13:49:06.201Z" }, + { url = "https://files.pythonhosted.org/packages/19/28/65e0980d668a6d44e699f59d1acf43d6b5d4893592c115ce7c680bb4dfa1/uuid_utils-0.17.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:387cf7437c94ddec08651a0f1081381299c7075bc48a6251d8922bf39973378a", size = 287858, upload-time = "2026-07-09T13:49:07.45Z" }, + { url = "https://files.pythonhosted.org/packages/8f/8d/5e97bcebc90fb6a10f98af3dc1ba552e04183aba59e2edc0b9cf486dd998/uuid_utils-0.17.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:220b52746d99e11964badac3c0869016e0c24bafb70a7dd5c2c072a6be3da9cc", size = 321587, upload-time = "2026-07-09T13:49:09.489Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d7/88b2a2370cc3d455ba0515fb6f5c8f7ac0c0f55a86801b6e56a432f22c17/uuid_utils-0.17.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0ab4a66e7a035ad6625cfc1fbdb34f5c2d25a80ae1ef4bfee458ea2036333c6d", size = 328964, upload-time = "2026-07-09T13:49:11.292Z" }, + { url = "https://files.pythonhosted.org/packages/bd/0f/181c5da673953dfc0958cb4fb3a4984a9098673ddb05cac68e994bc8511b/uuid_utils-0.17.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5641071337eb11d61a001ea08793bf72216f3241f0a433ed2764804b2a3e3cc7", size = 442909, upload-time = "2026-07-09T13:49:12.644Z" }, + { url = "https://files.pythonhosted.org/packages/ec/38/5c5e665af542884a8fd3c61725c38453239e13940326b5b70f3ef8881a97/uuid_utils-0.17.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9082e709014946b1f6e96ae6ecd93652efca2d2a6a3ab67dbe151c8b4bf193a4", size = 323076, upload-time = "2026-07-09T13:49:13.897Z" }, + { url = "https://files.pythonhosted.org/packages/f5/35/7de97de18cbf226c2a4f2104ad15e56ca4491717c81c0b71795c0c585b4e/uuid_utils-0.17.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1fd6f0e8a162dc0e9255b6aebe3cd175e76c33202f1bf39da9e6294b93db0099", size = 347360, upload-time = "2026-07-09T13:49:15.237Z" }, + { url = "https://files.pythonhosted.org/packages/26/a1/9915d5dd59fdd1957ded5d188c0ea0b9db5a1d84d42c8d8828a7b83b366e/uuid_utils-0.17.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d63010803d7c368963bbe6f7ec379593e76dd581d7db0f29118d88713c9e0354", size = 499267, upload-time = "2026-07-09T13:49:16.774Z" }, + { url = "https://files.pythonhosted.org/packages/c0/05/88108405262ec850cea0f95733445d6873e5772af3292baabd9ef8457740/uuid_utils-0.17.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a46bedc273b6f58f11dee816ff74999625ef8d007890f411b7a4975bf1c89330", size = 604940, upload-time = "2026-07-09T13:49:18.147Z" }, + { url = "https://files.pythonhosted.org/packages/89/d5/6dbcd300de47cc443cff2656cd5327a385751213dcb2101cfee7388170b2/uuid_utils-0.17.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:405233a5f625b3d995648f4647fa6befa4567cf3f74e1f6b9837e16f7310f0e0", size = 564172, upload-time = "2026-07-09T13:49:19.593Z" }, + { url = "https://files.pythonhosted.org/packages/ab/94/e8057f2288a415fba8a978bca4b589f5cb6b91a028a5dc07a1775938b33f/uuid_utils-0.17.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b6c5d2d71e1f17329150ad9427d27f4a3f29a01792e7ecdc64a98ac5368fc4d5", size = 528533, upload-time = "2026-07-09T13:49:21.075Z" }, + { url = "https://files.pythonhosted.org/packages/f0/6b/31713148c77e48e62f51aa042a98a54a8be0396912ea5130f83f52ae722d/uuid_utils-0.17.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:f7e9b8728ba07a3cb2f29d5aa1a266c2664eb8ef0fd43afa34627c92f7fac8f0", size = 99197, upload-time = "2026-07-09T13:49:22.351Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f3/ca6f6ac5428312df8ed632f6dd9f9e6aba23090471fcdeae53eab027e8b3/uuid_utils-0.17.0-cp314-cp314-win32.whl", hash = "sha256:58838921e377791ef22c64cc92141bfae030f43651ff9272f0f28a208a9e6a5a", size = 169540, upload-time = "2026-07-09T13:49:23.563Z" }, + { url = "https://files.pythonhosted.org/packages/c6/cd/7ede0db66411fa09817d79b680f7454ea9bee2d374e1922e4efd065760a3/uuid_utils-0.17.0-cp314-cp314-win_amd64.whl", hash = "sha256:42275ebd0e8e74e32cdbfb8bd88fc99576567d51d54a508020611fd8f4f463a0", size = 175984, upload-time = "2026-07-09T13:49:24.703Z" }, + { url = "https://files.pythonhosted.org/packages/f0/81/533b5f80cd4918c0693f4e1b7b90ceb1caa45f4266ae8b528135d7ecca5d/uuid_utils-0.17.0-cp314-cp314-win_arm64.whl", hash = "sha256:b5d11cccba076a32321ef1380dea956821f0b51794ef59df64e58fb1cd543aae", size = 174749, upload-time = "2026-07-09T13:49:25.886Z" }, + { url = "https://files.pythonhosted.org/packages/a0/13/f400ac39d06fd8be5b099c09e41bb975205926722a3e8d53348817cb7ff9/uuid_utils-0.17.0-cp314-cp314t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:fae8b282f0cb22a5de222999f7723f4e5ec04f6fcdf4aaef879b5b36625ae2b0", size = 562610, upload-time = "2026-07-09T13:49:27.374Z" }, + { url = "https://files.pythonhosted.org/packages/03/8c/c71c8312304c56f6d0bcba87cd402fa79bec35d18ffc8c41954196ca68e5/uuid_utils-0.17.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:967955620df45e6cffe2e9950cb9903cb455649396f896b26b04363a91a5054b", size = 289473, upload-time = "2026-07-09T13:49:28.989Z" }, + { url = "https://files.pythonhosted.org/packages/bb/cd/522117e2e5184ca1d4f0f85ee833e9e21bd8c6b99eff8a4d1a8e5a194e33/uuid_utils-0.17.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:375cde148430d60a4a07c03abaa0774c4fddfdd90de99b4ba02f24088bc9d750", size = 321600, upload-time = "2026-07-09T13:49:30.4Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f4/0d81f9bd346fc717bc561c08fa6457e0328966eb76e536b938fe77d56459/uuid_utils-0.17.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:975c17da26c5b9d46c336b03c52a057ac28378d6f9d98b58d32a038589bb3912", size = 329569, upload-time = "2026-07-09T13:49:31.732Z" }, + { url = "https://files.pythonhosted.org/packages/5e/41/26e1363f36a94c9e8ec2dd21d5f63088d3e7c723adbb12dcc8fdc77be417/uuid_utils-0.17.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3150d836290c88f1d26eb59c4db280d87417dd3bfaadd2889c77416c8f0ff6fa", size = 442051, upload-time = "2026-07-09T13:49:33.024Z" }, + { url = "https://files.pythonhosted.org/packages/2b/a7/2c1ed1b34d7df7fdcc11c28fd26d94d44843b37d9af2435ff9fd8abdbc08/uuid_utils-0.17.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9472a8de37faf8bd216c628e0e68c8f6bef730d3ba0a5060f3b0fa460c992ac2", size = 324372, upload-time = "2026-07-09T13:49:34.554Z" }, + { url = "https://files.pythonhosted.org/packages/78/bf/328d3c6bb22c496944a1b3b732207d71aa6964eb604e5e3b9dcb91ed0a00/uuid_utils-0.17.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d27c531edb8d1f38ca2eddaa1fa24913a460aeb721f2efd4ef42a124ce94e354", size = 348548, upload-time = "2026-07-09T13:49:35.898Z" }, + { url = "https://files.pythonhosted.org/packages/3e/76/a07de5cb7b90582fdbbc830fd19be129cbbb9897cfe239fef469d7bd2d09/uuid_utils-0.17.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5670c52a438e21483ce715776144914a4e2a2a5c62d9dee15f8a3e90cf128ae6", size = 498985, upload-time = "2026-07-09T13:49:37.142Z" }, + { url = "https://files.pythonhosted.org/packages/f4/62/9966e46ae34fcec6b06119631fb3c09705ea78835035ce3a82d3348eb61a/uuid_utils-0.17.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:6f29689a76fe7a49cbd629a794d0ec1eab48814e323a00a146a741b0195bde68", size = 605183, upload-time = "2026-07-09T13:49:38.648Z" }, + { url = "https://files.pythonhosted.org/packages/d7/4e/bb962ba0fe31e903b199f22cf4c1a6cba35a8987aef526d287277ab8ca8b/uuid_utils-0.17.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:4441600447d340ae103a353f01dbcd22ff680e5ee1a22988efe8d7b791d8fdb3", size = 565412, upload-time = "2026-07-09T13:49:40.115Z" }, + { url = "https://files.pythonhosted.org/packages/ce/9e/122adfeeeae8a84ccfd43bce627b104d12a2180a93bffd2c0e1b54dad7a6/uuid_utils-0.17.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e7b04935a79c03c41ad08d0a5f390aac968bfb561f1268897bc5b0f077971efd", size = 529885, upload-time = "2026-07-09T13:49:41.513Z" }, + { url = "https://files.pythonhosted.org/packages/b3/4f/257304dded339dc35fc9bf35722ac68fd4fdb930f255b8f7bccdf74ebba9/uuid_utils-0.17.0-cp314-cp314t-win32.whl", hash = "sha256:239d8a281fe10bae33205b5d43185834d556b18434e0a113b5dc1dfb2fd97e91", size = 169472, upload-time = "2026-07-09T13:49:42.871Z" }, + { url = "https://files.pythonhosted.org/packages/35/c8/e78c06db7e9ce317ce7b8759ff2058333eac75caa8c22b75f0059589c9be/uuid_utils-0.17.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e288a06cbbbcd01b44386e767985c9e21d2ad9bf59829aa7058d9a2a494804ab", size = 176271, upload-time = "2026-07-09T13:49:44.105Z" }, + { url = "https://files.pythonhosted.org/packages/a7/11/bd1c70e1ad3301163cebe66c8d26de26e6814d52f642a849448bd2833626/uuid_utils-0.17.0-cp314-cp314t-win_arm64.whl", hash = "sha256:1776a80d16369999b21627028cc5dbce819be83e1e079fdd7a51b587d2916db9", size = 175004, upload-time = "2026-07-09T13:49:45.591Z" }, +] + +[[package]] +name = "wcwidth" +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/74/c6428f875774288bec1396f5bfcbc2d925700a4dad61727fd5f2b12f249d/wcwidth-0.8.2.tar.gz", hash = "sha256:91fbef97204b96a3d4d421609b80340b760cf33e26da123ff243d76b1fda8dda", size = 1466253, upload-time = "2026-06-29T18:11:11.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl", hash = "sha256:d63947694a0539a1d51e01eda7caf800c291020e6cdd7e28ad7b14dd33ad4f85", size = 323166, upload-time = "2026-06-29T18:11:09.888Z" }, +] + +[[package]] +name = "webencodings" +version = "0.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47/webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923", size = 9721, upload-time = "2017-04-05T20:21:34.189Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", size = 11774, upload-time = "2017-04-05T20:21:32.581Z" }, +] + +[[package]] +name = "websockets" +version = "15.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/e6/26d09fab466b7ca9c7737474c52be4f76a40301b08362eb2dbc19dcc16c1/websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee", size = 177016, upload-time = "2025-03-05T20:03:41.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/6b/4545a0d843594f5d0771e86463606a3988b5a09ca5123136f8a76580dd63/websockets-15.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3e90baa811a5d73f3ca0bcbf32064d663ed81318ab225ee4f427ad4e26e5aff3", size = 175437, upload-time = "2025-03-05T20:02:16.706Z" }, + { url = "https://files.pythonhosted.org/packages/f4/71/809a0f5f6a06522af902e0f2ea2757f71ead94610010cf570ab5c98e99ed/websockets-15.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:592f1a9fe869c778694f0aa806ba0374e97648ab57936f092fd9d87f8bc03665", size = 173096, upload-time = "2025-03-05T20:02:18.832Z" }, + { url = "https://files.pythonhosted.org/packages/3d/69/1a681dd6f02180916f116894181eab8b2e25b31e484c5d0eae637ec01f7c/websockets-15.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2", size = 173332, upload-time = "2025-03-05T20:02:20.187Z" }, + { url = "https://files.pythonhosted.org/packages/a6/02/0073b3952f5bce97eafbb35757f8d0d54812b6174ed8dd952aa08429bcc3/websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b56bdcdb4505c8078cb6c7157d9811a85790f2f2b3632c7d1462ab5783d215", size = 183152, upload-time = "2025-03-05T20:02:22.286Z" }, + { url = "https://files.pythonhosted.org/packages/74/45/c205c8480eafd114b428284840da0b1be9ffd0e4f87338dc95dc6ff961a1/websockets-15.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0af68c55afbd5f07986df82831c7bff04846928ea8d1fd7f30052638788bc9b5", size = 182096, upload-time = "2025-03-05T20:02:24.368Z" }, + { url = "https://files.pythonhosted.org/packages/14/8f/aa61f528fba38578ec553c145857a181384c72b98156f858ca5c8e82d9d3/websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dee438fed052b52e4f98f76c5790513235efaa1ef7f3f2192c392cd7c91b65", size = 182523, upload-time = "2025-03-05T20:02:25.669Z" }, + { url = "https://files.pythonhosted.org/packages/ec/6d/0267396610add5bc0d0d3e77f546d4cd287200804fe02323797de77dbce9/websockets-15.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d5f6b181bb38171a8ad1d6aa58a67a6aa9d4b38d0f8c5f496b9e42561dfc62fe", size = 182790, upload-time = "2025-03-05T20:02:26.99Z" }, + { url = "https://files.pythonhosted.org/packages/02/05/c68c5adbf679cf610ae2f74a9b871ae84564462955d991178f95a1ddb7dd/websockets-15.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5d54b09eba2bada6011aea5375542a157637b91029687eb4fdb2dab11059c1b4", size = 182165, upload-time = "2025-03-05T20:02:30.291Z" }, + { url = "https://files.pythonhosted.org/packages/29/93/bb672df7b2f5faac89761cb5fa34f5cec45a4026c383a4b5761c6cea5c16/websockets-15.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be571a8b5afed347da347bfcf27ba12b069d9d7f42cb8c7028b5e98bbb12597", size = 182160, upload-time = "2025-03-05T20:02:31.634Z" }, + { url = "https://files.pythonhosted.org/packages/ff/83/de1f7709376dc3ca9b7eeb4b9a07b4526b14876b6d372a4dc62312bebee0/websockets-15.0.1-cp312-cp312-win32.whl", hash = "sha256:c338ffa0520bdb12fbc527265235639fb76e7bc7faafbb93f6ba80d9c06578a9", size = 176395, upload-time = "2025-03-05T20:02:33.017Z" }, + { url = "https://files.pythonhosted.org/packages/7d/71/abf2ebc3bbfa40f391ce1428c7168fb20582d0ff57019b69ea20fa698043/websockets-15.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcd5cf9e305d7b8338754470cf69cf81f420459dbae8a3b40cee57417f4614a7", size = 176841, upload-time = "2025-03-05T20:02:34.498Z" }, + { url = "https://files.pythonhosted.org/packages/cb/9f/51f0cf64471a9d2b4d0fc6c534f323b664e7095640c34562f5182e5a7195/websockets-15.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee443ef070bb3b6ed74514f5efaa37a252af57c90eb33b956d35c8e9c10a1931", size = 175440, upload-time = "2025-03-05T20:02:36.695Z" }, + { url = "https://files.pythonhosted.org/packages/8a/05/aa116ec9943c718905997412c5989f7ed671bc0188ee2ba89520e8765d7b/websockets-15.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5a939de6b7b4e18ca683218320fc67ea886038265fd1ed30173f5ce3f8e85675", size = 173098, upload-time = "2025-03-05T20:02:37.985Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0b/33cef55ff24f2d92924923c99926dcce78e7bd922d649467f0eda8368923/websockets-15.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:746ee8dba912cd6fc889a8147168991d50ed70447bf18bcda7039f7d2e3d9151", size = 173329, upload-time = "2025-03-05T20:02:39.298Z" }, + { url = "https://files.pythonhosted.org/packages/31/1d/063b25dcc01faa8fada1469bdf769de3768b7044eac9d41f734fd7b6ad6d/websockets-15.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:595b6c3969023ecf9041b2936ac3827e4623bfa3ccf007575f04c5a6aa318c22", size = 183111, upload-time = "2025-03-05T20:02:40.595Z" }, + { url = "https://files.pythonhosted.org/packages/93/53/9a87ee494a51bf63e4ec9241c1ccc4f7c2f45fff85d5bde2ff74fcb68b9e/websockets-15.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c714d2fc58b5ca3e285461a4cc0c9a66bd0e24c5da9911e30158286c9b5be7f", size = 182054, upload-time = "2025-03-05T20:02:41.926Z" }, + { url = "https://files.pythonhosted.org/packages/ff/b2/83a6ddf56cdcbad4e3d841fcc55d6ba7d19aeb89c50f24dd7e859ec0805f/websockets-15.0.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f3c1e2ab208db911594ae5b4f79addeb3501604a165019dd221c0bdcabe4db8", size = 182496, upload-time = "2025-03-05T20:02:43.304Z" }, + { url = "https://files.pythonhosted.org/packages/98/41/e7038944ed0abf34c45aa4635ba28136f06052e08fc2168520bb8b25149f/websockets-15.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:229cf1d3ca6c1804400b0a9790dc66528e08a6a1feec0d5040e8b9eb14422375", size = 182829, upload-time = "2025-03-05T20:02:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/e0/17/de15b6158680c7623c6ef0db361da965ab25d813ae54fcfeae2e5b9ef910/websockets-15.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:756c56e867a90fb00177d530dca4b097dd753cde348448a1012ed6c5131f8b7d", size = 182217, upload-time = "2025-03-05T20:02:50.14Z" }, + { url = "https://files.pythonhosted.org/packages/33/2b/1f168cb6041853eef0362fb9554c3824367c5560cbdaad89ac40f8c2edfc/websockets-15.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:558d023b3df0bffe50a04e710bc87742de35060580a293c2a984299ed83bc4e4", size = 182195, upload-time = "2025-03-05T20:02:51.561Z" }, + { url = "https://files.pythonhosted.org/packages/86/eb/20b6cdf273913d0ad05a6a14aed4b9a85591c18a987a3d47f20fa13dcc47/websockets-15.0.1-cp313-cp313-win32.whl", hash = "sha256:ba9e56e8ceeeedb2e080147ba85ffcd5cd0711b89576b83784d8605a7df455fa", size = 176393, upload-time = "2025-03-05T20:02:53.814Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6c/c65773d6cab416a64d191d6ee8a8b1c68a09970ea6909d16965d26bfed1e/websockets-15.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:e09473f095a819042ecb2ab9465aee615bd9c2028e4ef7d933600a8401c79561", size = 176837, upload-time = "2025-03-05T20:02:55.237Z" }, + { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, +] + +[[package]] +name = "win32-setctime" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/8f/705086c9d734d3b663af0e9bb3d4de6578d08f46b1b101c2442fd9aecaa2/win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0", size = 4867, upload-time = "2024-12-07T15:28:28.314Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083, upload-time = "2024-12-07T15:28:26.465Z" }, +] + +[[package]] +name = "wrapt" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/b0/c1f5a970721f06b85c0cd5142e0ff8fe067708abd779b0c4f4be7d61d09f/wrapt-2.3.0.tar.gz", hash = "sha256:681a2d0eefd721998f90642762b8e75c2159ec531b20ad5e437245ea7b06a107", size = 131509, upload-time = "2026-07-28T06:06:14.895Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/4a/d17a0fad1bf1c5f2c887ff71fef75654141b0880bff71d157d955b5bec3a/wrapt-2.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0a45ffae742ce91a16e11cb6c7cd71e7f9994f3cbd283b962ab093f5c6dcf525", size = 82139, upload-time = "2026-07-28T06:04:35.082Z" }, + { url = "https://files.pythonhosted.org/packages/6e/55/51b92daaf6defb57f4dc56bdcce985400f75c6984a03ca5e78ccac717028/wrapt-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:69e477046f2237ef0bc6547544ee73008dc764ca26eff44f09e976d221b34d5d", size = 82723, upload-time = "2026-07-28T06:04:36.502Z" }, + { url = "https://files.pythonhosted.org/packages/28/7f/cfd9bc4b1f5e424eeea83d0493e43f3b1b02707ce8e50c47945873982bd5/wrapt-2.3.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5d221a6e6ddd302b8397433184e96b59f259f50024b854db1c411a881586b6b8", size = 172381, upload-time = "2026-07-28T06:04:37.674Z" }, + { url = "https://files.pythonhosted.org/packages/cb/89/ff7814f6eb6856b479946117d1138a2fbb46cdb6b1f379db359056c69743/wrapt-2.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:392158c9a7f2ab1b8699418bfc0fe6f83548788c418b27d7bf2019ad3405cebb", size = 174120, upload-time = "2026-07-28T06:04:38.987Z" }, + { url = "https://files.pythonhosted.org/packages/12/1e/8eded8615d39e3ce81f626937a3a87b280a2a86239a2bf14a4b4bb345034/wrapt-2.3.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e5301c35cf75655eb33498f2bd6ae8703ca19940e3167dc9cdf740c712a39c60", size = 163035, upload-time = "2026-07-28T06:04:40.361Z" }, + { url = "https://files.pythonhosted.org/packages/35/ea/a0af2d9da62897af2a055484920de05dade30d2ba2c0d65cbdea875d3d8b/wrapt-2.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:418f54bb09d1762db02c7009b4051149893af3153a87f92d70356703c11eea02", size = 171887, upload-time = "2026-07-28T06:04:41.614Z" }, + { url = "https://files.pythonhosted.org/packages/7e/dd/63cd4c864c65ef4906df64bd2d378f4a62b54f28063f282dfb3bf93caead/wrapt-2.3.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:1598becd30f8f2777d18564064eb4f4dbe1ab0e05a8f09786d0ef505ac782bf3", size = 161113, upload-time = "2026-07-28T06:04:42.864Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ee/82f1fc9e431b5c2c5a6d201aa865dbeae3984c311c6d11a185f0c8367cf6/wrapt-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3da470536bf9645143323dd41b32db55c6f4304ad382094c1a1da8a92061e10d", size = 170530, upload-time = "2026-07-28T06:04:44.212Z" }, + { url = "https://files.pythonhosted.org/packages/37/a5/5dc590e863a419930d988f8b7ca3e75a6befcfb10b6003b3a152f3d5f732/wrapt-2.3.0-cp312-cp312-win32.whl", hash = "sha256:fb8e2e6704a1e0b1b989546c69e2688371ef4a07fa5f61bde3eb6211186f5ac1", size = 78323, upload-time = "2026-07-28T06:04:45.484Z" }, + { url = "https://files.pythonhosted.org/packages/51/f9/4a6925a07951df56394f7e6ebe14f69f1c5ef9d87aa63e0839acf15aa63a/wrapt-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:cdc021cb0b62471d6aac7f2bd92f3b4658073775f9ee7fcd325c511129e7bcc8", size = 81180, upload-time = "2026-07-28T06:04:47.021Z" }, + { url = "https://files.pythonhosted.org/packages/a8/4f/8b5de0395b2a72216751d41c9861df6facaeb611b619d8810ed2b3b23eb2/wrapt-2.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:67bfe2485f50368c3fcd2275fc1fd100e350d601e0058921a7c82678a465aeab", size = 80155, upload-time = "2026-07-28T06:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/8e/6e/0f88a072483e76b881e3fdcd6b6ffb4a5791002514fe541e72b1b73c859a/wrapt-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0d3fb71e65b001adfc42684522eeccd9c21d8ba679945abc993439567b66e59f", size = 81960, upload-time = "2026-07-28T06:04:49.622Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ff/b7e2776e7c294075eb712cc9ef573d1b818f393006d09787262b8fc871c4/wrapt-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:51a7a4181c1295774812271fbcd7c909df372bc25579d4ed9eb875caaf0ae86f", size = 82435, upload-time = "2026-07-28T06:04:50.9Z" }, + { url = "https://files.pythonhosted.org/packages/d8/90/343bb5d0f1f9669bc252a6073f085b4abf862511bd5c9c9eaec754341f1d/wrapt-2.3.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9045917809c63fdf7abe3a2ceaed3d670b8ee4500ddd9291192d30aeb34467c5", size = 170350, upload-time = "2026-07-28T06:04:52.187Z" }, + { url = "https://files.pythonhosted.org/packages/59/f8/13b79a392930bd0dd6b86cbfbfe1c40944110456e1dc6d809e5c46ece904/wrapt-2.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:54ca1d5573f69b5fe1d74f1f65799c68015e82f685efec9fd8cfa40a094c44d0", size = 170022, upload-time = "2026-07-28T06:04:53.599Z" }, + { url = "https://files.pythonhosted.org/packages/b2/fc/4f1b6918f5290db959d6e0c07f77385d87cede29c39c9cf8f145e9c82954/wrapt-2.3.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:242b60c21e30866e6a2fa606c612b47c553fa60c0eaeeeb7797fb842ac0ce609", size = 161043, upload-time = "2026-07-28T06:04:54.936Z" }, + { url = "https://files.pythonhosted.org/packages/01/e1/45d3cf74414780bdff6d0380467e003f6eb0f028b6c9403db868dbc7209c/wrapt-2.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3f3d7ec0a51fbfe00d3aef047641ff2c58b25565b4717fc1f90e050be01cba8", size = 168576, upload-time = "2026-07-28T06:04:56.261Z" }, + { url = "https://files.pythonhosted.org/packages/f3/73/2fa58dd97f191c997755e2c6d569a68f0c433db4e4b36099bdd7227b6cac/wrapt-2.3.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:261f53870cd4fb2bf38f9f972c56c728fd224cb7c65721307de59d9e7e6741ae", size = 159140, upload-time = "2026-07-28T06:04:57.754Z" }, + { url = "https://files.pythonhosted.org/packages/29/a8/08a56e2000a8816d449dcbad8c8b081697acbbd490821ceca0f9d8e8d20c/wrapt-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8159ec0b0cb7608175eb150de94c19e34f4d47ac655f5ca9baf45df6b688ffd3", size = 169263, upload-time = "2026-07-28T06:04:59.161Z" }, + { url = "https://files.pythonhosted.org/packages/9e/d4/354e1725e35a73b2af4fa70a3e024c7a5d1bf1802dfb862dcb668aae0253/wrapt-2.3.0-cp313-cp313-win32.whl", hash = "sha256:10461884b3014fbfc8eb7d09a93c5f246363e6711d9d881f95eb8c27fdef049f", size = 78241, upload-time = "2026-07-28T06:05:00.507Z" }, + { url = "https://files.pythonhosted.org/packages/6c/7e/34c87fa2174848dfee820322aaa318bab08913998ccecc8d2f57b4ad4639/wrapt-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:ac870cc97b73bb00ac353329e9559a4bebc47c4c86792ed9b23b58c15b6ad838", size = 81113, upload-time = "2026-07-28T06:05:01.839Z" }, + { url = "https://files.pythonhosted.org/packages/11/86/fcc9a530579e008c9478bb565a6cdfbfd33536660f069c8b91a6607c5050/wrapt-2.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:a65e8db2b4e90c2e7ade931086351c98ef420bf7a94ee08c95ac8a3cbbc43579", size = 80182, upload-time = "2026-07-28T06:05:03.152Z" }, + { url = "https://files.pythonhosted.org/packages/96/50/3864848b95b28ef73e17551fc8dccbff2628a834f52cf26a57f9c419fb83/wrapt-2.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:fd1f2f557dd3491fe75905e578f4db967393d40d1a8f468edc4d40ac7f2d5944", size = 83921, upload-time = "2026-07-28T06:05:04.476Z" }, + { url = "https://files.pythonhosted.org/packages/3b/4c/3d1921a60c3e8c71c540ff136e6a47a1fbccf7f671e818394889f7871d9c/wrapt-2.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:9f5d2aec29dfc76c37e23897dee92766a3fd4f3bff3ae7fc9c6b4bf37d8c1360", size = 84412, upload-time = "2026-07-28T06:05:05.921Z" }, + { url = "https://files.pythonhosted.org/packages/fa/1a/4a796ff7adb26ada6d4b758c94d47a38320b085e7099afc088efbbcdb006/wrapt-2.3.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:646d20d413ffcd1b0a2f700076e2d0252d872dcb7754860a73e45a59ea883614", size = 207168, upload-time = "2026-07-28T06:05:07.256Z" }, + { url = "https://files.pythonhosted.org/packages/1d/3e/d7777776806c579b761bac2f91721dda9f04c7a1b380213c5935cc750ae6/wrapt-2.3.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:379f670f45b7bb8993edd9f6fc36c6cc65edb81cffa0b504be34acb0303fff0a", size = 214351, upload-time = "2026-07-28T06:05:08.945Z" }, + { url = "https://files.pythonhosted.org/packages/63/27/2d64d394df7bf181955b3bb562bf33c4492fb4be113f53071106d43ad8b5/wrapt-2.3.0-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6208f302f110295d64b22a7ac96500c791bf492dce4366e622e4912b077c9687", size = 199020, upload-time = "2026-07-28T06:05:10.418Z" }, + { url = "https://files.pythonhosted.org/packages/3e/3d/fb31d3db7d9834d265fb1a27a2adf0ddf51557c67458c97b22439ad6ae3d/wrapt-2.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ed635a9ca4f3a5a2b900c10c69e823373bc00ebc114b459383596d3487da3570", size = 209969, upload-time = "2026-07-28T06:05:11.983Z" }, + { url = "https://files.pythonhosted.org/packages/1f/d1/8724b5da582e62070dc9bf4d8bf1972f317297eefd7ba1f2b5c6393ccf6c/wrapt-2.3.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:e3b9eaa742ae7a0aaaaad4ca4b69469d757af2d6e6663ef1dadc47adec0aeb41", size = 196324, upload-time = "2026-07-28T06:05:13.557Z" }, + { url = "https://files.pythonhosted.org/packages/0d/5c/3d9ef411149543016ee6bcf3af707f787cebd946527452b94bf122e9b7b4/wrapt-2.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d0f7284f88f4833705132d06d3b425a43095c2cbd07c58166aac3ab646ba12a4", size = 202610, upload-time = "2026-07-28T06:05:15.048Z" }, + { url = "https://files.pythonhosted.org/packages/13/9b/4fc042ceb757866dd4a5fc057b3b736f2b360d3703ce9f830d83dc9226e0/wrapt-2.3.0-cp313-cp313t-win32.whl", hash = "sha256:7ebb274aba688b043429eb1500ff8a76ce0cb8ac0812ca3e301f06247b8722b3", size = 79178, upload-time = "2026-07-28T06:05:16.469Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ff/b94878f8eed809ca042685276bcea9f24e8c2ca7c9653bb80bbb920a68a5/wrapt-2.3.0-cp313-cp313t-win_amd64.whl", hash = "sha256:c4bded758ad6f03b965830944a2f0bc5b2eb3767fe5a7310134315d1a6610e98", size = 82634, upload-time = "2026-07-28T06:05:18.026Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/663e1de5332a71685a729754312d327d4cada767c36e1c5a2db4c8de49e6/wrapt-2.3.0-cp313-cp313t-win_arm64.whl", hash = "sha256:d2cc64539da63e39ffb9c7ede849b6e8ddaaf7b3876b5cfb04efd85a5f3f4eb6", size = 81387, upload-time = "2026-07-28T06:05:19.417Z" }, + { url = "https://files.pythonhosted.org/packages/58/10/b073beaea89bc0d3670a75ff51139430a54b6af7ba7796507730634536dd/wrapt-2.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ea52a0d0f08c584943d5764be0e84efa912c8da23c23e1e285ff2f5641c18fcc", size = 81978, upload-time = "2026-07-28T06:05:21.133Z" }, + { url = "https://files.pythonhosted.org/packages/b3/31/0916d9cebf848ed3f1a0c1888faee421747df77331e4db2bc527a9a85988/wrapt-2.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fd85b0aa88efdb189d6ae2f35f4526943a8f091c38599c9c31478241c819e6a1", size = 82518, upload-time = "2026-07-28T06:05:22.562Z" }, + { url = "https://files.pythonhosted.org/packages/f5/73/31c1bf0f3384062751c2094dadb314916d70aa9b6bfd26d994b4a7b393fa/wrapt-2.3.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:141ed6211286a9660d8d6702de598b43f0934b4f0eda16393f100a80f501d945", size = 170187, upload-time = "2026-07-28T06:05:23.904Z" }, + { url = "https://files.pythonhosted.org/packages/ed/25/fce087d54b79b8905f3c3c9dd5f454bbd8d8acb80b960c4a6aee5b4659b3/wrapt-2.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e49885a62ec4ee854d1b9e6371fda6afd219917225752abf729a3f36d4df9a5", size = 169288, upload-time = "2026-07-28T06:05:25.378Z" }, + { url = "https://files.pythonhosted.org/packages/c7/30/0d09e6dddc6b7a7230ac77f50254b5980ab4fcd22976f72f8cc8a0404458/wrapt-2.3.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1d6159c9b2fefec02314e1332dbbbfaf960e369dfd26bcf7f8b258b5732065b3", size = 160932, upload-time = "2026-07-28T06:05:27.022Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ca/0913af0d2ec0c43865d32d615f518fea66c13c5c930e489e9b0de248e9a8/wrapt-2.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:24da48596326ef8e448cfa837b454f638713d3531262375f00e5a9681682fc07", size = 169017, upload-time = "2026-07-28T06:05:28.501Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f2/3d1e47ea81b822210f5df1bf942fd90780a75c055243d569b664529dea88/wrapt-2.3.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cd3a2edf0427013736b8127955cec62608c56e53ea47e82812ea32059cda407f", size = 159065, upload-time = "2026-07-28T06:05:30.01Z" }, + { url = "https://files.pythonhosted.org/packages/43/a5/ef2066ced8e5fca204e2b361e9708e36555b40949c583d997ea3b590817d/wrapt-2.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4fa0df3bff4e7ce45759f33fd39335fe2f60477bb9ecf7b8aa41e7d07ee36a23", size = 168821, upload-time = "2026-07-28T06:05:31.649Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e1/016104650d4e572fa91506eb396b3dd8efbccc9284fdc1c9479c3d21db28/wrapt-2.3.0-cp314-cp314-win32.whl", hash = "sha256:2935d5454b3f179a29b12cf390ee47246740ba2c3a7545b1b46ba31a5f2a4a0b", size = 78700, upload-time = "2026-07-28T06:05:33.391Z" }, + { url = "https://files.pythonhosted.org/packages/3d/97/6fdc20a9f2ca304748b3f0819cbf377d55260562777bf0b615431bc3c181/wrapt-2.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:cc2cea812e5cb179a796b766747e7d3b21088760d8deb95676d482b8c8e6fa7d", size = 81422, upload-time = "2026-07-28T06:05:34.774Z" }, + { url = "https://files.pythonhosted.org/packages/5e/a4/9cbd53bf05746bea2c392af39cb052427a8ec95cbd494d930733d8f44681/wrapt-2.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:22cc5c0a717bd4da87018ae0bffd4c19c6fb679d3ff357216ba566ab26c76cab", size = 80639, upload-time = "2026-07-28T06:05:36.228Z" }, + { url = "https://files.pythonhosted.org/packages/43/bb/6c5e4a0f66ea0d2b2dd267e8dd05a0014eea56840b3c8595d40b0a5d1f91/wrapt-2.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a6b5984cd65dd639546f0eb4b8eacf1c31cb2fe9fb5c27bffe240987cdb2cf84", size = 84030, upload-time = "2026-07-28T06:05:37.714Z" }, + { url = "https://files.pythonhosted.org/packages/6a/eb/a1aedf03283bc9cbf8a1783995ddc54e3c5a86878f19002d2c428494f4c5/wrapt-2.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c88abcf53daef80e01a75c7530e727fa6e2c1888fe83e3dcdba4c96216a1f5c7", size = 84419, upload-time = "2026-07-28T06:05:39.131Z" }, + { url = "https://files.pythonhosted.org/packages/63/61/50d511c0dc5105563849e86daa3e16ac7feef699f79fb05af45ea70107d5/wrapt-2.3.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:85de890ff968196e92dd1ae73a9fb8970495e7650a457b1c9ef0ac3dd550bce2", size = 207171, upload-time = "2026-07-28T06:05:40.69Z" }, + { url = "https://files.pythonhosted.org/packages/3f/59/9b538cf7795217e810699d16bc88b96a830d9b5c403eb2ec2db6b5f2ae81/wrapt-2.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50f416b74d092bb9f41b424e90dd457f365f7ba4b11de62a23679769a21bd85c", size = 214329, upload-time = "2026-07-28T06:05:42.287Z" }, + { url = "https://files.pythonhosted.org/packages/b3/28/9935d62b1499e5c8b3d191e99ba4eb31ca237a0b699142011a837e9dc7ea/wrapt-2.3.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39febbee6d77301d31da6996b152ce52452da7c7ef72aba10c2fa976dff9c295", size = 199079, upload-time = "2026-07-28T06:05:43.958Z" }, + { url = "https://files.pythonhosted.org/packages/2b/01/4446b80fa2ffa47a3449b250d004ba1c1937f07f64a179608fec735df866/wrapt-2.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:93513bec052c6cd987f9f580c3df068c8bc4ebae6543736be3ca7ec5959cafcd", size = 209992, upload-time = "2026-07-28T06:05:45.677Z" }, + { url = "https://files.pythonhosted.org/packages/d4/07/56f26c9f9979586a021e8148747004aba4498f49458c90b0502969b904e1/wrapt-2.3.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:729126e667da34d251b8ebf8a45ef0c5ddadc21542b3d6e1abf4259ece6508df", size = 196334, upload-time = "2026-07-28T06:05:47.608Z" }, + { url = "https://files.pythonhosted.org/packages/8b/41/6d7bcc895b0f28b2250e10908f060687b9165429dcd7f22ddb3d4c031b74/wrapt-2.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:626b69db2021aa01671ec7bbc9740e558522bd44c18cf2ce69bf3d666a014109", size = 202644, upload-time = "2026-07-28T06:05:49.183Z" }, + { url = "https://files.pythonhosted.org/packages/cd/25/7860927edba06b758b8852a6f02e832be715563c67a6795d94350bc81099/wrapt-2.3.0-cp314-cp314t-win32.whl", hash = "sha256:629d73378082c00a8173031f9fb30a3ac6abbc894a5bfdfae71fabc60642d501", size = 79685, upload-time = "2026-07-28T06:05:50.976Z" }, + { url = "https://files.pythonhosted.org/packages/c4/0f/270bafe92fde3b069a39bc01e39ee79340895b335640df861d43d2a51885/wrapt-2.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:42869085687f0aefd57c0f636c3f9354f8ffb321a8ba9cb52d19beb796e561c5", size = 83104, upload-time = "2026-07-28T06:05:52.405Z" }, + { url = "https://files.pythonhosted.org/packages/55/b3/af176d79a8515a8a720eccdad9a96f6e31a30abf2865430c8c42adf2fd13/wrapt-2.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b1e5aa486e269b00ed35e64771c7d0ab8096cfd2643405ca8cd60ebedc099a51", size = 81774, upload-time = "2026-07-28T06:05:53.902Z" }, + { url = "https://files.pythonhosted.org/packages/00/39/3daf9f47be208606586de4568ba6713db53ebc8fd7a575aea1fe57983b69/wrapt-2.3.0-py3-none-any.whl", hash = "sha256:d8c7ed08477429752b8c44991f40ad7838b18332a160698740a6bfbc10d998a2", size = 61866, upload-time = "2026-07-28T06:06:12.9Z" }, +] + +[[package]] +name = "xxhash" +version = "3.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/63/71aa56b151a1b28770037a61bd4e461c2619cfc8866a4fcaf1548605e325/xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46", size = 86223, upload-time = "2026-07-06T10:49:58.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/91/f65c34a7aa7b4e7cf4854f8e6ef3f7ee32ceac41d4f008da0780db0612f6/xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc", size = 34680, upload-time = "2026-07-06T10:44:40.932Z" }, + { url = "https://files.pythonhosted.org/packages/57/04/b10a245a4c09a9cfa88f8e9ae755029413ad1ac17047f9a61906e5ae0799/xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215", size = 32397, upload-time = "2026-07-06T10:44:42.196Z" }, + { url = "https://files.pythonhosted.org/packages/3a/75/45ab795b5945b6388583bd75202106af505537935566c15a1577797a0e08/xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478", size = 220549, upload-time = "2026-07-06T10:44:43.603Z" }, + { url = "https://files.pythonhosted.org/packages/13/44/5ba2bd0a14ddf4193fc7d8ec29625f659f22c06d60b28f04bf46305d8330/xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc", size = 241186, upload-time = "2026-07-06T10:44:45.534Z" }, + { url = "https://files.pythonhosted.org/packages/23/32/c4147def4d1e4538b906f82731e0ba23424377fc50a7cddd03cd284c8f63/xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8", size = 264852, upload-time = "2026-07-06T10:44:47.199Z" }, + { url = "https://files.pythonhosted.org/packages/6c/bd/71ed14f4f0318bb7fd7b2ec51999413487fa8da8d41208e84d50d1ef0f98/xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56", size = 242663, upload-time = "2026-07-06T10:44:48.846Z" }, + { url = "https://files.pythonhosted.org/packages/91/09/70af22c565a8473b3f2ae73f88e7721af281bc4a575236dbd1970c9f76f6/xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170", size = 473510, upload-time = "2026-07-06T10:44:50.695Z" }, + { url = "https://files.pythonhosted.org/packages/18/96/34db781c8f0cf99c544ca1f2bc2e5bf55426e1eb4ca6de8ea5da56a9f352/xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc", size = 220469, upload-time = "2026-07-06T10:44:52.422Z" }, + { url = "https://files.pythonhosted.org/packages/93/5f/9a184f615fa5a4dce30c01534f62946ce5a11ce40f73785cbd356ccabaa9/xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c", size = 310290, upload-time = "2026-07-06T10:44:54.142Z" }, + { url = "https://files.pythonhosted.org/packages/a9/dc/9b9a9789011ee153723a5eb9e7dd7fcbae2ba9b3fe7a729249ca7c252056/xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d", size = 238173, upload-time = "2026-07-06T10:44:55.693Z" }, + { url = "https://files.pythonhosted.org/packages/ec/4d/71c6005ada9dcb608a4e1902e8475ecadb5f3fbfa04e1e244d276a2d0c43/xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb", size = 269026, upload-time = "2026-07-06T10:44:57.424Z" }, + { url = "https://files.pythonhosted.org/packages/2f/87/d6c036ba25dfbd9c8633be5aa86fc9474bbb9e2c68212a841d090abe7344/xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061", size = 224970, upload-time = "2026-07-06T10:44:59.085Z" }, + { url = "https://files.pythonhosted.org/packages/48/62/4c1f035a41c5752aa05e195b6c904c07b94fe9061a16de61e72a6e6b135f/xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887", size = 240820, upload-time = "2026-07-06T10:45:00.746Z" }, + { url = "https://files.pythonhosted.org/packages/da/14/d39d565069b87e86d21a2af2a31d04db79249d25aa8d5b62959056a89857/xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e", size = 300619, upload-time = "2026-07-06T10:45:02.716Z" }, + { url = "https://files.pythonhosted.org/packages/13/22/75467acc887edc8cf71c97ab1708feb3df7a88bda589b9f399765c6387d2/xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975", size = 443267, upload-time = "2026-07-06T10:45:04.653Z" }, + { url = "https://files.pythonhosted.org/packages/a4/b6/1da3baa5fa6ef705e3425fddd382be7dfc4dfba2686df90a20f16e9c7b1b/xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a", size = 217338, upload-time = "2026-07-06T10:45:06.304Z" }, + { url = "https://files.pythonhosted.org/packages/78/dd/b5295a9f97484e7a1c2b283a742ca45e3104991c55a1ef670dde161829ba/xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e", size = 31970, upload-time = "2026-07-06T10:45:07.823Z" }, + { url = "https://files.pythonhosted.org/packages/ec/31/3fa0b807d7e21515cd975e7fe5c039d52ac3e9401a96d6ad68dae6305215/xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724", size = 32741, upload-time = "2026-07-06T10:45:09.42Z" }, + { url = "https://files.pythonhosted.org/packages/b8/05/86feada74e239600e6875aa507afb40482a89b92700aa74a92da83bdcb77/xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396", size = 29234, upload-time = "2026-07-06T10:45:10.809Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8c/446bb782cd0d27007a917b5569a08dd73219c3e8d6e459014db104b27bdb/xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913", size = 38562, upload-time = "2026-07-06T10:45:12.425Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ec/c0c45627eaa6be7a5d6117423adf8f7a15b17ee74b4b17072cca5959a225/xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec", size = 36656, upload-time = "2026-07-06T10:45:13.932Z" }, + { url = "https://files.pythonhosted.org/packages/f6/94/8324c04cc7597154caaeba6c094e01fbd2e7601d01e7a13eea9f5420e77b/xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297", size = 31169, upload-time = "2026-07-06T10:45:15.687Z" }, + { url = "https://files.pythonhosted.org/packages/40/a4/beb6bb26e1184e126dbe7a5682330214ef54dcfbf882078aa9f4b5428d42/xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed", size = 32177, upload-time = "2026-07-06T10:45:17.035Z" }, + { url = "https://files.pythonhosted.org/packages/56/0f/fc4c92a5a528f839b34b6419b2e53c8597f2a629d5a1f5d721f65bfa1fd6/xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276", size = 34642, upload-time = "2026-07-06T10:45:18.39Z" }, + { url = "https://files.pythonhosted.org/packages/d4/58/edbfb141d4000767ac6a9694f8ac0763e2c2e983e65c9e31620ba56e2667/xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0", size = 34684, upload-time = "2026-07-06T10:45:20.033Z" }, + { url = "https://files.pythonhosted.org/packages/07/3f/5072f1f0f5714186f0ac2a0b5a4929ce30d4b845e94886b6c01b6ebda0be/xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315", size = 32401, upload-time = "2026-07-06T10:45:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/49/c7/802ea2f9c2ed59219934d6d65c470d502b1788043eae277a52af8658bda6/xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907", size = 220617, upload-time = "2026-07-06T10:45:23.234Z" }, + { url = "https://files.pythonhosted.org/packages/99/a8/e10488efd31fcb13fcd6acbc6e788f10c6f8e3a0cc4ae3eb89dc19c55a12/xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0", size = 241295, upload-time = "2026-07-06T10:45:25.364Z" }, + { url = "https://files.pythonhosted.org/packages/18/cc/14180b17d44892a631f8ae7323c30bfbb1328efc8209e528a480293528ac/xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2", size = 264688, upload-time = "2026-07-06T10:45:27.09Z" }, + { url = "https://files.pythonhosted.org/packages/a9/72/a14019d0c5f6c41ee407a503036ae32787c91325ca218a96a9b5627be651/xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329", size = 242740, upload-time = "2026-07-06T10:45:28.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/08/92550e556c6fcfcb96c6a336945eb53a431ed43120ed749636debb16c5cf/xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0", size = 473599, upload-time = "2026-07-06T10:45:30.524Z" }, + { url = "https://files.pythonhosted.org/packages/29/83/e361d3c1acd1b21e1d489616de6fa4aaf843365d8179f612e3743eac20a9/xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437", size = 220559, upload-time = "2026-07-06T10:45:32.979Z" }, + { url = "https://files.pythonhosted.org/packages/05/01/006a4243c2c2a6831827f9999f6d1c23feeef100eb023c1f886022a00bf3/xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62", size = 310383, upload-time = "2026-07-06T10:45:35.875Z" }, + { url = "https://files.pythonhosted.org/packages/d8/20/af388e8bf9f9a0f89eeef7d2a1935d176ee1c20bc6adeda05035879379cf/xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf", size = 238228, upload-time = "2026-07-06T10:45:38.02Z" }, + { url = "https://files.pythonhosted.org/packages/63/6b/4666579a87eebd1744663c404297355fa0658617b015cedfa58810ee7036/xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf", size = 269137, upload-time = "2026-07-06T10:45:39.954Z" }, + { url = "https://files.pythonhosted.org/packages/de/d3/e963a8a46f900a137d91b02144d8ea07a8f812971b138204a3b2f8b8e55c/xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3", size = 225068, upload-time = "2026-07-06T10:45:41.718Z" }, + { url = "https://files.pythonhosted.org/packages/aa/80/9d181dbcde4b0fe48375f48833a5832d4b8cd2b349b15110c92ee472d874/xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104", size = 240874, upload-time = "2026-07-06T10:45:43.414Z" }, + { url = "https://files.pythonhosted.org/packages/39/15/ce3ab5a1cd27ead25a5196e55a7284220f6ad6e316da494ffd900b2b600f/xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e", size = 300702, upload-time = "2026-07-06T10:45:45.135Z" }, + { url = "https://files.pythonhosted.org/packages/96/c0/2281a8ab5f2a62dbf57a23c58a01ccc1d98abf40f71193c8a81f59e759b5/xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893", size = 443351, upload-time = "2026-07-06T10:45:47.188Z" }, + { url = "https://files.pythonhosted.org/packages/81/2e/071a58c1a53a52d4f7a3aa0987be0c396dffd40da8204805fe1b130a81f4/xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3", size = 217396, upload-time = "2026-07-06T10:45:48.925Z" }, + { url = "https://files.pythonhosted.org/packages/68/44/36ab58134badd9d3433fc7b53c4ca8d113d8e807782885628640f8297a4d/xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a", size = 31974, upload-time = "2026-07-06T10:45:50.591Z" }, + { url = "https://files.pythonhosted.org/packages/96/2a/2a0b84798448e766f7b89ceed073cb0cb5a43fc9ebbacbdea74a38de18e3/xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55", size = 32739, upload-time = "2026-07-06T10:45:52.221Z" }, + { url = "https://files.pythonhosted.org/packages/d4/60/bb51dbf7c363ff88a7cbd50b7959718219577ef44d7cf255929ffc4a2194/xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a", size = 29239, upload-time = "2026-07-06T10:45:53.714Z" }, + { url = "https://files.pythonhosted.org/packages/56/d3/827ca123c2ee5443a6aaed3c5dd199237dc2f010e2bebd7ec09ef36f3a5f/xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc", size = 34964, upload-time = "2026-07-06T10:45:55.535Z" }, + { url = "https://files.pythonhosted.org/packages/05/67/67ae2a3ccdeb8b8ef025d35aee9edd1d26c3abe5051d47da9286232afbf8/xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92", size = 32697, upload-time = "2026-07-06T10:45:57.288Z" }, + { url = "https://files.pythonhosted.org/packages/38/5a/3d3994346e1f45493679cb5c1ffc2bf454e410e9d1e8a662d253becee91e/xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75", size = 225954, upload-time = "2026-07-06T10:45:58.897Z" }, + { url = "https://files.pythonhosted.org/packages/3f/2c/53169270309b7cd8e05504e07fe123bac053b89d00ac63617faacf0a2ec0/xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda", size = 249776, upload-time = "2026-07-06T10:46:00.977Z" }, + { url = "https://files.pythonhosted.org/packages/70/e0/5c551d8d592f944506f7c5185e210255c15e672a3c6008c156a1bd9b775e/xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629", size = 274776, upload-time = "2026-07-06T10:46:02.869Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2a/d3a762270cee2d7bcd0e25e28c623e5f3f5c0dc637b66e3e47dd5b0bb3f0/xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068", size = 252056, upload-time = "2026-07-06T10:46:04.688Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/b78e4373b2cb6d1c42af60ea2d7e9146ad0710b239ac7f706d5d31d5bb98/xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1", size = 482108, upload-time = "2026-07-06T10:46:06.498Z" }, + { url = "https://files.pythonhosted.org/packages/e6/0d/642d923336ea61a15f8ce64fc7e078729e6e06c3a026e517fa79b2c23b7a/xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647", size = 226739, upload-time = "2026-07-06T10:46:08.598Z" }, + { url = "https://files.pythonhosted.org/packages/a6/0a/a37d6da6427d45a8d23e3ee3a0ca9c9d4a90364849c6637fe2963a755f9b/xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1", size = 319658, upload-time = "2026-07-06T10:46:10.504Z" }, + { url = "https://files.pythonhosted.org/packages/4a/51/ebbd40da8a3f1bc53b4b7a9a87f8e28bd95c5f21bc14b8a57860cf367d1b/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113", size = 246059, upload-time = "2026-07-06T10:46:12.634Z" }, + { url = "https://files.pythonhosted.org/packages/24/4c/d9014030147e1f0bb26e7da47aa240dd9ec61c763c573e558111d869f8e1/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9", size = 275535, upload-time = "2026-07-06T10:46:14.614Z" }, + { url = "https://files.pythonhosted.org/packages/84/86/caee2db41fadcd5a25aa4323213f9afec5a8586d4e419241e3d659362bd7/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa", size = 231292, upload-time = "2026-07-06T10:46:16.452Z" }, + { url = "https://files.pythonhosted.org/packages/0b/60/f52f08bcdc904c4514ea5c25caa19e9f3214144434a6ff96dc82dc1cbddd/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31", size = 250490, upload-time = "2026-07-06T10:46:18.318Z" }, + { url = "https://files.pythonhosted.org/packages/24/a0/94dc7ae310838f250669c6ad7168e6d6fca17d49dac1053f06dc232c4a56/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6", size = 309861, upload-time = "2026-07-06T10:46:20.503Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f9/adeead7d0eb28cdfc2832544ea639ffbc6749ccde47a8e228d667459182e/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398", size = 448739, upload-time = "2026-07-06T10:46:22.79Z" }, + { url = "https://files.pythonhosted.org/packages/04/a4/22ec0e07db57d901c9298ae98aa3cf2be45bafded6f07c13131e85b89032/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838", size = 223657, upload-time = "2026-07-06T10:46:24.831Z" }, + { url = "https://files.pythonhosted.org/packages/94/32/8a9531f37b59e5a013003db7cb7414baf4ce7e0e1268e0d5947cd3d6a2df/xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22", size = 32377, upload-time = "2026-07-06T10:46:26.86Z" }, + { url = "https://files.pythonhosted.org/packages/e7/ab/2ca45fd7f671de5f81fc297ef1c95080b40c86ec6be0cc6034b8f7707ac8/xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629", size = 33274, upload-time = "2026-07-06T10:46:28.39Z" }, + { url = "https://files.pythonhosted.org/packages/5a/54/20d7163463ddb6438b73a427d1655a77a502cf9b9b0c3ada3599629d9c0a/xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65", size = 29375, upload-time = "2026-07-06T10:46:29.962Z" }, + { url = "https://files.pythonhosted.org/packages/c2/8b/df2ba04f22a6cd6b39f96a6577329a8471a55c90ef8d8e2f7c102363613f/xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba", size = 38430, upload-time = "2026-07-06T10:46:31.496Z" }, + { url = "https://files.pythonhosted.org/packages/b2/4f/6a059e8ad3ca8deedc91dfe335b211204900895152212c03ebbe721de68b/xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17", size = 36558, upload-time = "2026-07-06T10:46:33.078Z" }, + { url = "https://files.pythonhosted.org/packages/cb/95/40be178205acce092ae418feb20ac737b32a02c7b864926ed0717354c9f8/xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd", size = 31181, upload-time = "2026-07-06T10:46:34.793Z" }, + { url = "https://files.pythonhosted.org/packages/3f/89/2da4dbf051bafa156c0e3f12012db2b0ac3b84ff37ca1f021f6bfffcdfbb/xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5", size = 32192, upload-time = "2026-07-06T10:46:36.393Z" }, + { url = "https://files.pythonhosted.org/packages/7c/4e/e000bbae3566bc8e0be771a8a0f294aa99075e3f0bc4ef43922ebffdebc8/xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf", size = 34691, upload-time = "2026-07-06T10:46:38.1Z" }, + { url = "https://files.pythonhosted.org/packages/b4/4a/ea954aacc7d1c8711880ac2b55da94429a9b4296b151c4fc0966549ca1ee/xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9", size = 34807, upload-time = "2026-07-06T10:46:39.647Z" }, + { url = "https://files.pythonhosted.org/packages/ca/29/df598e738ff37558ac627264deb2e560902d9bf7f46d3bd5175c9eee593e/xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711", size = 32410, upload-time = "2026-07-06T10:46:41.359Z" }, + { url = "https://files.pythonhosted.org/packages/59/9c/81ab40e7d33ada0b3df5d1bc884894d15dbf4f805cd645b685e4606bb8e0/xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920", size = 220564, upload-time = "2026-07-06T10:46:43.463Z" }, + { url = "https://files.pythonhosted.org/packages/fd/6f/62ae6f5c8606320a0e2a41c2dc8c6d91cc5d63d0f84dd9582e9543779dd8/xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc", size = 241462, upload-time = "2026-07-06T10:46:45.509Z" }, + { url = "https://files.pythonhosted.org/packages/15/a1/9c3a0ec6cb524396f551eddd102a76690a795494eb9784fc67542b0daa37/xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62", size = 264491, upload-time = "2026-07-06T10:46:47.655Z" }, + { url = "https://files.pythonhosted.org/packages/64/f2/700a4674e4308eb59d2fdb973977e82eae231bea5044753fee5c9eec0e0c/xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f", size = 242905, upload-time = "2026-07-06T10:46:49.857Z" }, + { url = "https://files.pythonhosted.org/packages/f3/8a/72d9874375c8d4cbc64a8cd1d659d5695a8765c3db82efa82dc5bd9f14d0/xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e", size = 473873, upload-time = "2026-07-06T10:46:51.953Z" }, + { url = "https://files.pythonhosted.org/packages/03/f0/6db07590ed7e0a77f186ef0bcea8d52553bf1ba57833e09467a2411f0f2d/xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b", size = 220765, upload-time = "2026-07-06T10:46:55.41Z" }, + { url = "https://files.pythonhosted.org/packages/8f/10/00d12d8b8beabbf49a8bbc626fb9f40445145a8887eb41a6acfb69149ac4/xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d", size = 310478, upload-time = "2026-07-06T10:46:57.729Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f9/12a82394eefb0f185d15a7f7b9f627c61c475a72dd83718436a5b84b42ac/xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b", size = 238393, upload-time = "2026-07-06T10:46:59.87Z" }, + { url = "https://files.pythonhosted.org/packages/20/f3/53f963e320b9ce678337aa7273f39ce692ded8b99e3d22a866ec722159ab/xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc", size = 268704, upload-time = "2026-07-06T10:47:01.806Z" }, + { url = "https://files.pythonhosted.org/packages/0a/50/5b5badbd87c82d9f9b5f58ac74a3f29ef08f6fc387b324b8fd482450b862/xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f", size = 225015, upload-time = "2026-07-06T10:47:03.784Z" }, + { url = "https://files.pythonhosted.org/packages/30/93/3ca68265afe7b4e69435e08a7b6a1d9d0f2a071e889da1f8041ed00fe878/xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1", size = 240951, upload-time = "2026-07-06T10:47:05.816Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a6/27e19670c40f46b5e76e11f2f4713d21054804568425d870670e757172ad/xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e", size = 300751, upload-time = "2026-07-06T10:47:07.95Z" }, + { url = "https://files.pythonhosted.org/packages/bc/fb/b33e27689959fe7ed2ae0b830af41560d65213943983afa9db3a8d481bce/xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231", size = 443480, upload-time = "2026-07-06T10:47:10Z" }, + { url = "https://files.pythonhosted.org/packages/26/60/0e0d973be5fe280753ef02fbc89349492ad6e903bf1dcb870b668f94b662/xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe", size = 217657, upload-time = "2026-07-06T10:47:12.196Z" }, + { url = "https://files.pythonhosted.org/packages/ad/68/c9e3ecef4a9a417d464cb5bd200aa12f73192dee677901b9e08e0ad0d1bb/xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab", size = 32690, upload-time = "2026-07-06T10:47:14.204Z" }, + { url = "https://files.pythonhosted.org/packages/d7/99/e9e44588c0b62837bbec5ba7927816de0afa03406b1a0b6c7a7e1d1a30a0/xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f", size = 33460, upload-time = "2026-07-06T10:47:16.021Z" }, + { url = "https://files.pythonhosted.org/packages/45/2b/64f36d86380b3657ad9031967ab814f3ef31307174650853f69c18932ebc/xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5", size = 30092, upload-time = "2026-07-06T10:47:17.81Z" }, + { url = "https://files.pythonhosted.org/packages/92/cb/18b64bff88c58a0ca209dc533e63cf02d7ae5aa6b1b9a9fd14e81b5dbd60/xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c", size = 35024, upload-time = "2026-07-06T10:47:19.821Z" }, + { url = "https://files.pythonhosted.org/packages/af/1d/72d8a70520e5dcddb472ea0486d299da3240745a10658290cd7b5690ede2/xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230", size = 32697, upload-time = "2026-07-06T10:47:21.649Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b8/e041f555903c56db3d0a731b3d72a6575d75e0ed868b1bd2e5176111ca44/xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489", size = 226044, upload-time = "2026-07-06T10:47:23.612Z" }, + { url = "https://files.pythonhosted.org/packages/3a/7e/5cdcf06bf6ec4b5d2ac073feb23432ec1d603fd438864cbd2c09c7cb45e1/xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5", size = 249899, upload-time = "2026-07-06T10:47:25.812Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c0/eb7e059cb5e1dba11fd30d2fdf882f56e5a417a3eaa43669d43623767f45/xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e", size = 274892, upload-time = "2026-07-06T10:47:27.931Z" }, + { url = "https://files.pythonhosted.org/packages/66/74/a600aaf7cd39957fd1510adeedb1749c1e7eb82bd632a1153d9c664c3135/xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6", size = 252243, upload-time = "2026-07-06T10:47:30.288Z" }, + { url = "https://files.pythonhosted.org/packages/ad/04/78d88fa75a6763e5d09bf1b947a392a27988903381b219006f92f3c68fc8/xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339", size = 482191, upload-time = "2026-07-06T10:47:32.45Z" }, + { url = "https://files.pythonhosted.org/packages/7f/06/07a8aea1108d682de8791ce608cdf367d75ff4e7e57cd3c154bdc6f47b23/xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542", size = 226877, upload-time = "2026-07-06T10:47:34.705Z" }, + { url = "https://files.pythonhosted.org/packages/ed/b5/86bade5618a524d2c06c4041aa2fe8e5749ce16e88afba60d67c1684a21f/xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08", size = 319794, upload-time = "2026-07-06T10:47:37.08Z" }, + { url = "https://files.pythonhosted.org/packages/23/69/9b1a2b89b1621bb740fbcb7beb512f60f99480c1bdc680c0c90e1f56ff75/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775", size = 246202, upload-time = "2026-07-06T10:47:39.676Z" }, + { url = "https://files.pythonhosted.org/packages/08/ea/662ed6cb49f1d34078b6a3a3e0f3d29ff93fd7b5a03c0bc9ecfd9b2159c3/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8", size = 275628, upload-time = "2026-07-06T10:47:41.99Z" }, + { url = "https://files.pythonhosted.org/packages/13/f5/49fc9e4c6728a5a3bd8fe639199d2fa67609b3a84f938aff6e8568dd3e4f/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e", size = 231390, upload-time = "2026-07-06T10:47:44.233Z" }, + { url = "https://files.pythonhosted.org/packages/64/9d/3acaf8f599c0e0b30e910a3a11ba32929da53c86dc73c7c55fe6a010b4e9/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99", size = 250600, upload-time = "2026-07-06T10:47:47.611Z" }, + { url = "https://files.pythonhosted.org/packages/23/64/8acab4c5ec60dbe664b5b9858fd44c2413b07e535b09556a0a5022e78aa6/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0", size = 310032, upload-time = "2026-07-06T10:47:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/56/47/a0288d7329b1fe63e2734a32d19d444a96ae2b4810f545bc61e561224917/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07", size = 448882, upload-time = "2026-07-06T10:47:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/01/e7/3071dfd3beb5c38204ce1cf56bf7749fce08de900fa92714b81d1d8ca1f2/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799", size = 223728, upload-time = "2026-07-06T10:47:55.093Z" }, + { url = "https://files.pythonhosted.org/packages/12/11/b99949f0ba2b07e9f9ffe83b9c86faa685f9080725dc21a916a607313be5/xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497", size = 33150, upload-time = "2026-07-06T10:47:57.274Z" }, + { url = "https://files.pythonhosted.org/packages/54/1c/09703eb341f8416e74e58d6c6732d4b5c46de59c942363203cb237cc95b0/xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684", size = 34005, upload-time = "2026-07-06T10:47:59.434Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f9/6ed7251bb6a8af10ac73b1821c60583d2826e5b2064e45a979c935287c98/xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20", size = 30239, upload-time = "2026-07-06T10:48:01.874Z" }, +] + +[[package]] +name = "yarl" +version = "1.24.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "multidict" }, + { name = "propcache" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/33/ebe9e3d1f86c7a0b51094c0a146392045ca1631d2664889539dec8088a33/yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f", size = 228679, upload-time = "2026-07-20T02:07:45.435Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/84/71d051c850b5af41d168c679d9eb67eb7c55283ac4ee131673edf134bc4e/yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d", size = 136035, upload-time = "2026-07-20T02:05:25.489Z" }, + { url = "https://files.pythonhosted.org/packages/03/4d/8ad27f9a1b7e69313cca5d695b925b48efe51208d3490e0844bae97cabc0/yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec", size = 97642, upload-time = "2026-07-20T02:05:27.429Z" }, + { url = "https://files.pythonhosted.org/packages/ea/b4/05b4131c407006cd1e410e9c6539f16a0945724677e5364447313c15ea3e/yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c", size = 97323, upload-time = "2026-07-20T02:05:29.441Z" }, + { url = "https://files.pythonhosted.org/packages/20/16/e618c875c73e0e39611f20a581b3d5e8d59b8857bf001bee3263044c6deb/yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54", size = 107741, upload-time = "2026-07-20T02:05:31.367Z" }, + { url = "https://files.pythonhosted.org/packages/d9/9a/c4defeaf3ed33fcb346aacf9c6e971a8d4e2bde04a0310e79abb208e7965/yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12", size = 103570, upload-time = "2026-07-20T02:05:33.303Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e7/0e0e0de5865ebd5914537ef486f36c727a59865c3ac0cf5ff1b32aececbf/yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d", size = 115815, upload-time = "2026-07-20T02:05:35.292Z" }, + { url = "https://files.pythonhosted.org/packages/2b/27/ca56b700cb170aba25a3893b75355b213935657dc5714d2383354a270e62/yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1", size = 116025, upload-time = "2026-07-20T02:05:37.503Z" }, + { url = "https://files.pythonhosted.org/packages/d6/d0/d56c859b8222116f5d68459199f48359e0bf121b6f65a69bf329b3602ba0/yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9", size = 109835, upload-time = "2026-07-20T02:05:39.506Z" }, + { url = "https://files.pythonhosted.org/packages/70/a2/3a35557e4d1a79425040eba202ccaf08bdc8717680fc77e2498a1ad2e0a5/yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027", size = 108884, upload-time = "2026-07-20T02:05:41.584Z" }, + { url = "https://files.pythonhosted.org/packages/e4/35/ef4c26356b7913c68983bac2d72a4212b3347af551cb8d250b99b5ed7b7f/yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b", size = 107308, upload-time = "2026-07-20T02:05:43.697Z" }, + { url = "https://files.pythonhosted.org/packages/d5/91/ff0dc66c2ccf3e0153ab97ff61eabab4400e6a5264af427ab30cd69f1857/yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293", size = 103646, upload-time = "2026-07-20T02:05:45.895Z" }, + { url = "https://files.pythonhosted.org/packages/74/f0/33b9271c7f881766359d58266fa0811d2e5210ed860e28da7dc6d7786344/yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e", size = 115305, upload-time = "2026-07-20T02:05:47.832Z" }, + { url = "https://files.pythonhosted.org/packages/ef/65/fd79fb1868c4a80db8661091de525bf430f63c3bea1b20e8b6a84fc7d359/yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b", size = 108404, upload-time = "2026-07-20T02:05:49.604Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ba/dbabe6b262f17a816c70cfc09558dbf03ece3ec76684d02f911a3d3a189c/yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce", size = 115940, upload-time = "2026-07-20T02:05:51.741Z" }, + { url = "https://files.pythonhosted.org/packages/a5/43/fab2d1dad9d340a268cdde63756a123d069723efff6a372d123fa74a9517/yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba", size = 110006, upload-time = "2026-07-20T02:05:53.554Z" }, + { url = "https://files.pythonhosted.org/packages/c4/27/41eb51bbd1b8d89546b83897cfb0164f1e109304fd408dbb151b639eec0f/yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b", size = 97618, upload-time = "2026-07-20T02:05:55.57Z" }, + { url = "https://files.pythonhosted.org/packages/3c/25/b2553764b3d65db711d8f45416351ec4f420847558eb669edcbcaadf5780/yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c", size = 93018, upload-time = "2026-07-20T02:05:57.554Z" }, + { url = "https://files.pythonhosted.org/packages/e1/63/64ef361967cc983573149dc1515d531db5da8a4c92d22bb833d59e01b313/yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2", size = 135075, upload-time = "2026-07-20T02:05:59.671Z" }, + { url = "https://files.pythonhosted.org/packages/bb/89/55920fd853ce43e608adbc3962456f0d649d6bb15250dc2988321da0fe1c/yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb", size = 97225, upload-time = "2026-07-20T02:06:01.769Z" }, + { url = "https://files.pythonhosted.org/packages/15/f0/7688d3f2cfff7590df2af38ec46d969f4281a4dddb08a9ad2eafbcdddf98/yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075", size = 96751, upload-time = "2026-07-20T02:06:03.676Z" }, + { url = "https://files.pythonhosted.org/packages/05/1a/a851a0f94aaaf379dd4f901bfc80f634280bec51eb260b47363e2a4cd62e/yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff", size = 107960, upload-time = "2026-07-20T02:06:05.699Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a8/faea066c12f9c77ca0de90641f1655f9dd7b412477bf28c76d692f3aecff/yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448", size = 103500, upload-time = "2026-07-20T02:06:07.556Z" }, + { url = "https://files.pythonhosted.org/packages/fb/9c/1e67084c2a6e2f2db0e3be798328cb3be42c0119b621d25461479a224d21/yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f", size = 115780, upload-time = "2026-07-20T02:06:09.599Z" }, + { url = "https://files.pythonhosted.org/packages/58/86/1f94664e147474337e3359f52012cf3d02f825f694317b178bfba1078c62/yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd", size = 115308, upload-time = "2026-07-20T02:06:11.352Z" }, + { url = "https://files.pythonhosted.org/packages/0a/43/8e55ae7538ba5f28ccb3c845c6dd4549cf7016d5992e5326512519107cdd/yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16", size = 110574, upload-time = "2026-07-20T02:06:13.129Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ba/a889ec8765cedcf2ac44dcb02d6a21e4861399b243b263c5f2dde27ee740/yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213", size = 109914, upload-time = "2026-07-20T02:06:15.243Z" }, + { url = "https://files.pythonhosted.org/packages/9c/c3/e45f821af67b791c2dbbe4a9f4137a1d33f8d386654a05a0c3f47bdfa25d/yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24", size = 107712, upload-time = "2026-07-20T02:06:17.443Z" }, + { url = "https://files.pythonhosted.org/packages/02/00/2ab0f42c9857fcb490bfaa6647b14540b53d241ab209f23220b958cc5832/yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385", size = 104251, upload-time = "2026-07-20T02:06:19.259Z" }, + { url = "https://files.pythonhosted.org/packages/7a/70/709d9a286e98af2c7fd8e4e6cada658b5c0e30d87dd7e2a63c2fb5767217/yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c", size = 115319, upload-time = "2026-07-20T02:06:21.207Z" }, + { url = "https://files.pythonhosted.org/packages/5c/6c/3eaa515142991fe84cfc483ff986492211f1978f90161ccefdbec919d09b/yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4", size = 109163, upload-time = "2026-07-20T02:06:23.006Z" }, + { url = "https://files.pythonhosted.org/packages/bb/64/711dafce66c323a3144d470547a71c5384c57623308ac8bb5e4b903ac148/yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144", size = 115435, upload-time = "2026-07-20T02:06:24.923Z" }, + { url = "https://files.pythonhosted.org/packages/cf/f3/9b9d0e6d84bea851eb1ba99e4bdc755b86fd813e49ec86dfe42f26befdef/yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4", size = 110691, upload-time = "2026-07-20T02:06:26.973Z" }, + { url = "https://files.pythonhosted.org/packages/86/e4/62a06b7e87c4246ac76b7c2da136f972eb4a3a1fc94abb07e7022d6fdb0a/yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740", size = 97454, upload-time = "2026-07-20T02:06:29.163Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c9/5fc8025b318ab10db413b61056bd0d95c557a70e8df4210c7511f866329c/yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1", size = 92813, upload-time = "2026-07-20T02:06:31.113Z" }, + { url = "https://files.pythonhosted.org/packages/a9/08/5f3085fef9564217074db9dd8573de1795bc82cde61a7ad10b6a7234a569/yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76", size = 135680, upload-time = "2026-07-20T02:06:33.273Z" }, + { url = "https://files.pythonhosted.org/packages/98/35/ba9436e579bd48a8801f2021d842d9ab4994c26e4c7dd3a4c1f1bcb57a9e/yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d", size = 97395, upload-time = "2026-07-20T02:06:35.259Z" }, + { url = "https://files.pythonhosted.org/packages/18/a9/a07f76f3c44e02b25cc743af5ef93eef27f7013eadca770451b6a6ccb5db/yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75", size = 97223, upload-time = "2026-07-20T02:06:37.216Z" }, + { url = "https://files.pythonhosted.org/packages/77/f7/a9a1d6fa7dd9e388f95b30f6ad3ec4e285f6c8f61f44ce16070c3fcfe414/yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9", size = 108777, upload-time = "2026-07-20T02:06:39.292Z" }, + { url = "https://files.pythonhosted.org/packages/2f/44/e0b86c302471fabd6f02808ecf2ac52b8412b624787849d4bf2cdb466f6f/yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede", size = 103119, upload-time = "2026-07-20T02:06:41.456Z" }, + { url = "https://files.pythonhosted.org/packages/d1/16/9c16d180bf8faaf223225eb50e1245870ff1ae0e302a27153988e65c51fd/yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca", size = 116471, upload-time = "2026-07-20T02:06:43.696Z" }, + { url = "https://files.pythonhosted.org/packages/d2/8d/b219b9df28a02ce95cfbdd41d2f7caa5669d0ff979c1c9975697145e33c5/yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027", size = 115974, upload-time = "2026-07-20T02:06:45.874Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e8/f20557aca240d88e69850ad1ee91756821d094bb1310565c04d25c6682a2/yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9", size = 110830, upload-time = "2026-07-20T02:06:47.852Z" }, + { url = "https://files.pythonhosted.org/packages/db/18/199b85109a53eeca64ee19c9cca228287e8e4ab0cc1a09b28f530e65cce0/yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41", size = 110054, upload-time = "2026-07-20T02:06:49.84Z" }, + { url = "https://files.pythonhosted.org/packages/aa/2f/ed28147f8cd7f48c49367c90713b30a555284b6105a6a56f3a05568da795/yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373", size = 108312, upload-time = "2026-07-20T02:06:51.835Z" }, + { url = "https://files.pythonhosted.org/packages/c5/c5/55e16ae0a5c227cea8df1c6871ba57d614a34243146c05729caf2a1bd9c5/yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36", size = 103662, upload-time = "2026-07-20T02:06:54.061Z" }, + { url = "https://files.pythonhosted.org/packages/8d/ea/dbd7c2caec459c9a426f18b02688ecbfb58620d0f6a3422d24769fbaf8ab/yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0", size = 116090, upload-time = "2026-07-20T02:06:56.015Z" }, + { url = "https://files.pythonhosted.org/packages/06/84/39ce4ce3059e07fece5fbdbee8c4053406af9aca911ce9fa5f8548aab6af/yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5", size = 109523, upload-time = "2026-07-20T02:06:57.926Z" }, + { url = "https://files.pythonhosted.org/packages/a9/8b/71ff44137b405c64a7788075669c24010019f57a7464b78c3a6cbee539d9/yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5", size = 116084, upload-time = "2026-07-20T02:06:59.868Z" }, + { url = "https://files.pythonhosted.org/packages/62/c0/423078fdd4042e1862c11f0ffd977a0ffa393783c12bee94685923bc189e/yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4", size = 111006, upload-time = "2026-07-20T02:07:01.907Z" }, + { url = "https://files.pythonhosted.org/packages/cf/52/6daa2ee9d95e5c98b8128f8df91eb692eb423ab274b8cf08db52152fad26/yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad", size = 99215, upload-time = "2026-07-20T02:07:03.852Z" }, + { url = "https://files.pythonhosted.org/packages/ec/0e/464a847d7359e0da75dd9fc5c1d1aa35d0159ea31e5f8e66a3c1c29ff3d0/yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f", size = 94566, upload-time = "2026-07-20T02:07:06.074Z" }, + { url = "https://files.pythonhosted.org/packages/e2/55/e03acc4446772660bc335e86e41ef31e4d0d838fd641531a11a5ee33b493/yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88", size = 142533, upload-time = "2026-07-20T02:07:08.284Z" }, + { url = "https://files.pythonhosted.org/packages/ae/71/4acd3a1fc7cf14345cdb302665ecd2097f62c365b4f14ca17d4f37775cf9/yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba", size = 100776, upload-time = "2026-07-20T02:07:10.197Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0b/cfb76b7fe99686db264bff829779a539d923e7564ffd7ef18da6c54c3774/yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928", size = 100913, upload-time = "2026-07-20T02:07:12.357Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3f/7116e782992abbd4fb6948488aec72078895e929a23078290739e8396fce/yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f", size = 106507, upload-time = "2026-07-20T02:07:14.173Z" }, + { url = "https://files.pythonhosted.org/packages/33/90/d4d2d73ee78229cc889872eb8e085d8f5c6f51abdb178409fd9b23cf74fd/yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95", size = 99219, upload-time = "2026-07-20T02:07:16.019Z" }, + { url = "https://files.pythonhosted.org/packages/3e/fa/a6df1a9bccd644eec00abee0dff4277416222cec435330fd1f2858523ec1/yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc", size = 111804, upload-time = "2026-07-20T02:07:18.141Z" }, + { url = "https://files.pythonhosted.org/packages/8a/9e/7b2a1f4bcc20e9447156dd2b1c4d01f70d9df0759025ee7d09a84ffae134/yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da", size = 110943, upload-time = "2026-07-20T02:07:20.06Z" }, + { url = "https://files.pythonhosted.org/packages/08/ff/22c92affb0f9b623ca753d27d968b5625b868f12c6378d049d55ae247643/yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a", size = 108251, upload-time = "2026-07-20T02:07:22.217Z" }, + { url = "https://files.pythonhosted.org/packages/45/44/5769b96298c1e195fb412997b6090af2a84105cf59c17613558a2d011d1f/yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0", size = 106025, upload-time = "2026-07-20T02:07:24.083Z" }, + { url = "https://files.pythonhosted.org/packages/4c/40/009e8e791fd9762c0e1567e69248acb4f49064597e1680874c16dd8bb798/yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498", size = 106573, upload-time = "2026-07-20T02:07:26.248Z" }, + { url = "https://files.pythonhosted.org/packages/20/c6/b7480578f8a0a80946f36ad6df547ecec704f9ba69d2de60f8aa6f1c1cbf/yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104", size = 100751, upload-time = "2026-07-20T02:07:28.098Z" }, + { url = "https://files.pythonhosted.org/packages/d4/27/4476f3360b91a48c5cf125e91f59a3bd35299d84a431a258d57f5977bb11/yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331", size = 111643, upload-time = "2026-07-20T02:07:30.88Z" }, + { url = "https://files.pythonhosted.org/packages/4c/4b/5cdd3e5ee944e8af31e52f6cd3d3af5fd7b937e036ccbbba2c9ffebede95/yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550", size = 106312, upload-time = "2026-07-20T02:07:33.06Z" }, + { url = "https://files.pythonhosted.org/packages/18/86/f406b0c2a6f99575de2da671ef47aa06f89a5be83a27a46971c3b86cecdb/yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6", size = 110379, upload-time = "2026-07-20T02:07:35.155Z" }, + { url = "https://files.pythonhosted.org/packages/f0/6c/9f3adfbd3b30b4fa0f7ccb3a83eba2c1152d3fff554d535e640ba0f7ba2b/yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047", size = 108497, upload-time = "2026-07-20T02:07:37.35Z" }, + { url = "https://files.pythonhosted.org/packages/dd/37/91eb2e5ca883a529c1b390348a74cd9fc0512171727f547ce70bfe02be5c/yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104", size = 102450, upload-time = "2026-07-20T02:07:39.578Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f4/ed5c402ac8fde4403ed3366c2716bfddc8a6677ebd59f3d62772cc7fe468/yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688", size = 97222, upload-time = "2026-07-20T02:07:41.55Z" }, + { url = "https://files.pythonhosted.org/packages/61/02/962c1cbfc401a30c1d034dc67ff395f64b52302c6d62de556c1fca99acc0/yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7", size = 58612, upload-time = "2026-07-20T02:07:43.461Z" }, +] + +[[package]] +name = "zss" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1e/d1/ed34d12f55d07cc1efb61d74fb2f64f46a705557f5bdd1ef1b810f0e2ec5/zss-1.2.0.tar.gz", hash = "sha256:07bb937441929ccb82961f4f7b80fbce9e2b20d0e46ddcbcbc1fcb094f585b50", size = 9790, upload-time = "2018-03-12T15:02:20.208Z" } + +[[package]] +name = "zstandard" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/fc/f26eb6ef91ae723a03e16eddb198abcfce2bc5a42e224d44cc8b6765e57e/zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b", size = 795738, upload-time = "2025-09-14T22:16:56.237Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1c/d920d64b22f8dd028a8b90e2d756e431a5d86194caa78e3819c7bf53b4b3/zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00", size = 640436, upload-time = "2025-09-14T22:16:57.774Z" }, + { url = "https://files.pythonhosted.org/packages/53/6c/288c3f0bd9fcfe9ca41e2c2fbfd17b2097f6af57b62a81161941f09afa76/zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64", size = 5343019, upload-time = "2025-09-14T22:16:59.302Z" }, + { url = "https://files.pythonhosted.org/packages/1e/15/efef5a2f204a64bdb5571e6161d49f7ef0fffdbca953a615efbec045f60f/zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea", size = 5063012, upload-time = "2025-09-14T22:17:01.156Z" }, + { url = "https://files.pythonhosted.org/packages/b7/37/a6ce629ffdb43959e92e87ebdaeebb5ac81c944b6a75c9c47e300f85abdf/zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb", size = 5394148, upload-time = "2025-09-14T22:17:03.091Z" }, + { url = "https://files.pythonhosted.org/packages/e3/79/2bf870b3abeb5c070fe2d670a5a8d1057a8270f125ef7676d29ea900f496/zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a", size = 5451652, upload-time = "2025-09-14T22:17:04.979Z" }, + { url = "https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902", size = 5546993, upload-time = "2025-09-14T22:17:06.781Z" }, + { url = "https://files.pythonhosted.org/packages/85/c7/3483ad9ff0662623f3648479b0380d2de5510abf00990468c286c6b04017/zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f", size = 5046806, upload-time = "2025-09-14T22:17:08.415Z" }, + { url = "https://files.pythonhosted.org/packages/08/b3/206883dd25b8d1591a1caa44b54c2aad84badccf2f1de9e2d60a446f9a25/zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b", size = 5576659, upload-time = "2025-09-14T22:17:10.164Z" }, + { url = "https://files.pythonhosted.org/packages/9d/31/76c0779101453e6c117b0ff22565865c54f48f8bd807df2b00c2c404b8e0/zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6", size = 4953933, upload-time = "2025-09-14T22:17:11.857Z" }, + { url = "https://files.pythonhosted.org/packages/18/e1/97680c664a1bf9a247a280a053d98e251424af51f1b196c6d52f117c9720/zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91", size = 5268008, upload-time = "2025-09-14T22:17:13.627Z" }, + { url = "https://files.pythonhosted.org/packages/1e/73/316e4010de585ac798e154e88fd81bb16afc5c5cb1a72eeb16dd37e8024a/zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708", size = 5433517, upload-time = "2025-09-14T22:17:16.103Z" }, + { url = "https://files.pythonhosted.org/packages/5b/60/dd0f8cfa8129c5a0ce3ea6b7f70be5b33d2618013a161e1ff26c2b39787c/zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512", size = 5814292, upload-time = "2025-09-14T22:17:17.827Z" }, + { url = "https://files.pythonhosted.org/packages/fc/5f/75aafd4b9d11b5407b641b8e41a57864097663699f23e9ad4dbb91dc6bfe/zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa", size = 5360237, upload-time = "2025-09-14T22:17:19.954Z" }, + { url = "https://files.pythonhosted.org/packages/ff/8d/0309daffea4fcac7981021dbf21cdb2e3427a9e76bafbcdbdf5392ff99a4/zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd", size = 436922, upload-time = "2025-09-14T22:17:24.398Z" }, + { url = "https://files.pythonhosted.org/packages/79/3b/fa54d9015f945330510cb5d0b0501e8253c127cca7ebe8ba46a965df18c5/zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01", size = 506276, upload-time = "2025-09-14T22:17:21.429Z" }, + { url = "https://files.pythonhosted.org/packages/ea/6b/8b51697e5319b1f9ac71087b0af9a40d8a6288ff8025c36486e0c12abcc4/zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9", size = 462679, upload-time = "2025-09-14T22:17:23.147Z" }, + { url = "https://files.pythonhosted.org/packages/35/0b/8df9c4ad06af91d39e94fa96cc010a24ac4ef1378d3efab9223cc8593d40/zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94", size = 795735, upload-time = "2025-09-14T22:17:26.042Z" }, + { url = "https://files.pythonhosted.org/packages/3f/06/9ae96a3e5dcfd119377ba33d4c42a7d89da1efabd5cb3e366b156c45ff4d/zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1", size = 640440, upload-time = "2025-09-14T22:17:27.366Z" }, + { url = "https://files.pythonhosted.org/packages/d9/14/933d27204c2bd404229c69f445862454dcc101cd69ef8c6068f15aaec12c/zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f", size = 5343070, upload-time = "2025-09-14T22:17:28.896Z" }, + { url = "https://files.pythonhosted.org/packages/6d/db/ddb11011826ed7db9d0e485d13df79b58586bfdec56e5c84a928a9a78c1c/zstandard-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea", size = 5063001, upload-time = "2025-09-14T22:17:31.044Z" }, + { url = "https://files.pythonhosted.org/packages/db/00/87466ea3f99599d02a5238498b87bf84a6348290c19571051839ca943777/zstandard-0.25.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e", size = 5394120, upload-time = "2025-09-14T22:17:32.711Z" }, + { url = "https://files.pythonhosted.org/packages/2b/95/fc5531d9c618a679a20ff6c29e2b3ef1d1f4ad66c5e161ae6ff847d102a9/zstandard-0.25.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551", size = 5451230, upload-time = "2025-09-14T22:17:34.41Z" }, + { url = "https://files.pythonhosted.org/packages/63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a", size = 5547173, upload-time = "2025-09-14T22:17:36.084Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d5/ba05ed95c6b8ec30bd468dfeab20589f2cf709b5c940483e31d991f2ca58/zstandard-0.25.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611", size = 5046736, upload-time = "2025-09-14T22:17:37.891Z" }, + { url = "https://files.pythonhosted.org/packages/50/d5/870aa06b3a76c73eced65c044b92286a3c4e00554005ff51962deef28e28/zstandard-0.25.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3", size = 5576368, upload-time = "2025-09-14T22:17:40.206Z" }, + { url = "https://files.pythonhosted.org/packages/5d/35/398dc2ffc89d304d59bc12f0fdd931b4ce455bddf7038a0a67733a25f550/zstandard-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b", size = 4954022, upload-time = "2025-09-14T22:17:41.879Z" }, + { url = "https://files.pythonhosted.org/packages/9a/5c/36ba1e5507d56d2213202ec2b05e8541734af5f2ce378c5d1ceaf4d88dc4/zstandard-0.25.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851", size = 5267889, upload-time = "2025-09-14T22:17:43.577Z" }, + { url = "https://files.pythonhosted.org/packages/70/e8/2ec6b6fb7358b2ec0113ae202647ca7c0e9d15b61c005ae5225ad0995df5/zstandard-0.25.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250", size = 5433952, upload-time = "2025-09-14T22:17:45.271Z" }, + { url = "https://files.pythonhosted.org/packages/7b/01/b5f4d4dbc59ef193e870495c6f1275f5b2928e01ff5a81fecb22a06e22fb/zstandard-0.25.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98", size = 5814054, upload-time = "2025-09-14T22:17:47.08Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e5/fbd822d5c6f427cf158316d012c5a12f233473c2f9c5fe5ab1ae5d21f3d8/zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf", size = 5360113, upload-time = "2025-09-14T22:17:48.893Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e0/69a553d2047f9a2c7347caa225bb3a63b6d7704ad74610cb7823baa08ed7/zstandard-0.25.0-cp313-cp313-win32.whl", hash = "sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09", size = 436936, upload-time = "2025-09-14T22:17:52.658Z" }, + { url = "https://files.pythonhosted.org/packages/d9/82/b9c06c870f3bd8767c201f1edbdf9e8dc34be5b0fbc5682c4f80fe948475/zstandard-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5", size = 506232, upload-time = "2025-09-14T22:17:50.402Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/60c3c01243bb81d381c9916e2a6d9e149ab8627c0c7d7abb2d73384b3c0c/zstandard-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049", size = 462671, upload-time = "2025-09-14T22:17:51.533Z" }, + { url = "https://files.pythonhosted.org/packages/3d/5c/f8923b595b55fe49e30612987ad8bf053aef555c14f05bb659dd5dbe3e8a/zstandard-0.25.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3", size = 795887, upload-time = "2025-09-14T22:17:54.198Z" }, + { url = "https://files.pythonhosted.org/packages/8d/09/d0a2a14fc3439c5f874042dca72a79c70a532090b7ba0003be73fee37ae2/zstandard-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f", size = 640658, upload-time = "2025-09-14T22:17:55.423Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7c/8b6b71b1ddd517f68ffb55e10834388d4f793c49c6b83effaaa05785b0b4/zstandard-0.25.0-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c", size = 5379849, upload-time = "2025-09-14T22:17:57.372Z" }, + { url = "https://files.pythonhosted.org/packages/a4/86/a48e56320d0a17189ab7a42645387334fba2200e904ee47fc5a26c1fd8ca/zstandard-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439", size = 5058095, upload-time = "2025-09-14T22:17:59.498Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ad/eb659984ee2c0a779f9d06dbfe45e2dc39d99ff40a319895df2d3d9a48e5/zstandard-0.25.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043", size = 5551751, upload-time = "2025-09-14T22:18:01.618Z" }, + { url = "https://files.pythonhosted.org/packages/61/b3/b637faea43677eb7bd42ab204dfb7053bd5c4582bfe6b1baefa80ac0c47b/zstandard-0.25.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859", size = 6364818, upload-time = "2025-09-14T22:18:03.769Z" }, + { url = "https://files.pythonhosted.org/packages/31/dc/cc50210e11e465c975462439a492516a73300ab8caa8f5e0902544fd748b/zstandard-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0", size = 5560402, upload-time = "2025-09-14T22:18:05.954Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ae/56523ae9c142f0c08efd5e868a6da613ae76614eca1305259c3bf6a0ed43/zstandard-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7", size = 4955108, upload-time = "2025-09-14T22:18:07.68Z" }, + { url = "https://files.pythonhosted.org/packages/98/cf/c899f2d6df0840d5e384cf4c4121458c72802e8bda19691f3b16619f51e9/zstandard-0.25.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2", size = 5269248, upload-time = "2025-09-14T22:18:09.753Z" }, + { url = "https://files.pythonhosted.org/packages/1b/c0/59e912a531d91e1c192d3085fc0f6fb2852753c301a812d856d857ea03c6/zstandard-0.25.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344", size = 5430330, upload-time = "2025-09-14T22:18:11.966Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/7e31db1240de2df22a58e2ea9a93fc6e38cc29353e660c0272b6735d6669/zstandard-0.25.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c", size = 5811123, upload-time = "2025-09-14T22:18:13.907Z" }, + { url = "https://files.pythonhosted.org/packages/f6/49/fac46df5ad353d50535e118d6983069df68ca5908d4d65b8c466150a4ff1/zstandard-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088", size = 5359591, upload-time = "2025-09-14T22:18:16.465Z" }, + { url = "https://files.pythonhosted.org/packages/c2/38/f249a2050ad1eea0bb364046153942e34abba95dd5520af199aed86fbb49/zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12", size = 444513, upload-time = "2025-09-14T22:18:20.61Z" }, + { url = "https://files.pythonhosted.org/packages/3a/43/241f9615bcf8ba8903b3f0432da069e857fc4fd1783bd26183db53c4804b/zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2", size = 516118, upload-time = "2025-09-14T22:18:17.849Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ef/da163ce2450ed4febf6467d77ccb4cd52c4c30ab45624bad26ca0a27260c/zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d", size = 476940, upload-time = "2025-09-14T22:18:19.088Z" }, +] From b0a9009e64bf35607b6764b94d92e3714b16cfb0 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 06:49:15 +0530 Subject: [PATCH 02/38] feat(bench_core): unified provider/model abstraction with tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 1 of the harness redesign. Collapses the N×M per-provider runner forks into one tested core: declared capabilities + merge chain, a single reasoning translation (the six divergent Gemini-floor encodings + OpenAI/Fireworks/ Anthropic/OpenRouter), media block builders (fixes the audio_url silent-drop and the RGBA→JPEG crash centrally), response decode, error classification, the openai-compat/anthropic/gemini adapters, a self-healing fallback ladder that emits capability hints, the shared async harness (rate-limit/semaphore/retry/ resume/incremental checkpoint), and one result contract with content-based discovery. 8 target YAMLs across all providers; 106 offline tests. Schema note: reasoning keys are off_value/on_value, not off/on — bare off:/on: are YAML 1.1 booleans (off: low -> {False: low}); guarded by a test. --- bench_core/__init__.py | 0 bench_core/capabilities.py | 146 ++++++++++++++++ bench_core/config.py | 196 ++++++++++++++++++++++ bench_core/decode.py | 69 ++++++++ bench_core/errors.py | 101 +++++++++++ bench_core/execute.py | 110 ++++++++++++ bench_core/media.py | 112 +++++++++++++ bench_core/providers/__init__.py | 0 bench_core/providers/anthropic.py | 82 +++++++++ bench_core/providers/base.py | 51 ++++++ bench_core/providers/gemini.py | 72 ++++++++ bench_core/providers/openai_compat.py | 70 ++++++++ bench_core/reasoning.py | 109 ++++++++++++ bench_core/request.py | 53 ++++++ bench_core/response.py | 17 ++ bench_core/results.py | 99 +++++++++++ bench_core/runner.py | 137 +++++++++++++++ bench_core/targets/claude-sonnet-4-6.yaml | 5 + bench_core/targets/gemini-2.5-flash.yaml | 11 ++ bench_core/targets/gemini-3.7-flash.yaml | 5 + bench_core/targets/gpt-5.5.yaml | 5 + bench_core/targets/grok-4.3.yaml | 11 ++ bench_core/targets/inkling-small.yaml | 10 ++ bench_core/targets/inkling.yaml | 9 + bench_core/targets/interfaze-beta.yaml | 5 + bench_core/targets/kimi-k2.6.yaml | 13 ++ tests/__init__.py | 0 tests/test_capabilities.py | 74 ++++++++ tests/test_config.py | 121 +++++++++++++ tests/test_decode.py | 86 ++++++++++ tests/test_encode_anthropic.py | 71 ++++++++ tests/test_encode_gemini.py | 59 +++++++ tests/test_encode_openai_compat.py | 122 ++++++++++++++ tests/test_errors.py | 75 +++++++++ tests/test_execute.py | 162 ++++++++++++++++++ tests/test_media.py | 118 +++++++++++++ tests/test_reasoning.py | 168 +++++++++++++++++++ tests/test_results.py | 57 +++++++ tests/test_runner.py | 143 ++++++++++++++++ tests/test_targets.py | 45 +++++ 40 files changed, 2799 insertions(+) create mode 100644 bench_core/__init__.py create mode 100644 bench_core/capabilities.py create mode 100644 bench_core/config.py create mode 100644 bench_core/decode.py create mode 100644 bench_core/errors.py create mode 100644 bench_core/execute.py create mode 100644 bench_core/media.py create mode 100644 bench_core/providers/__init__.py create mode 100644 bench_core/providers/anthropic.py create mode 100644 bench_core/providers/base.py create mode 100644 bench_core/providers/gemini.py create mode 100644 bench_core/providers/openai_compat.py create mode 100644 bench_core/reasoning.py create mode 100644 bench_core/request.py create mode 100644 bench_core/response.py create mode 100644 bench_core/results.py create mode 100644 bench_core/runner.py create mode 100644 bench_core/targets/claude-sonnet-4-6.yaml create mode 100644 bench_core/targets/gemini-2.5-flash.yaml create mode 100644 bench_core/targets/gemini-3.7-flash.yaml create mode 100644 bench_core/targets/gpt-5.5.yaml create mode 100644 bench_core/targets/grok-4.3.yaml create mode 100644 bench_core/targets/inkling-small.yaml create mode 100644 bench_core/targets/inkling.yaml create mode 100644 bench_core/targets/interfaze-beta.yaml create mode 100644 bench_core/targets/kimi-k2.6.yaml create mode 100644 tests/__init__.py create mode 100644 tests/test_capabilities.py create mode 100644 tests/test_config.py create mode 100644 tests/test_decode.py create mode 100644 tests/test_encode_anthropic.py create mode 100644 tests/test_encode_gemini.py create mode 100644 tests/test_encode_openai_compat.py create mode 100644 tests/test_errors.py create mode 100644 tests/test_execute.py create mode 100644 tests/test_media.py create mode 100644 tests/test_reasoning.py create mode 100644 tests/test_results.py create mode 100644 tests/test_runner.py create mode 100644 tests/test_targets.py diff --git a/bench_core/__init__.py b/bench_core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bench_core/capabilities.py b/bench_core/capabilities.py new file mode 100644 index 0000000..23530b6 --- /dev/null +++ b/bench_core/capabilities.py @@ -0,0 +1,146 @@ +"""Declared model capabilities + the merge that resolves them. + +Every axis the audit found varying across providers/models is a field here, so +that a new model is described by *data* (a target YAML file) rather than a new +code path. Provider adapters ship a default capability dict; the target file +overrides fields; the CLI can override again. `deep_merge` collapses the chain; +`Capabilities.from_dict` gives adapters a typed, validated view. +""" + +from __future__ import annotations + +import copy +from collections.abc import Mapping +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + + +class ReasoningStyle(str, Enum): + """How a host expresses "think this much". + + The vocabulary is genuinely different per host — this enum names the + mechanism so the adapter can translate an abstract mode into the concrete + param without a benchmark-level `if provider == ...`. + """ + + EFFORT = ( + "effort" # OpenAI/Fireworks/Interfaze: reasoning_effort="none|low|high|..." + ) + THINKING_LEVEL = "thinking_level" # Gemini 3.x: thinking_config.thinking_level enum + THINKING_BUDGET = ( + "thinking_budget" # Gemini 2.5: thinking_config.thinking_budget int + ) + DISABLED_BLOCK = ( + "disabled_block" # Anthropic: thinking={"type":"disabled"|"enabled"} + ) + REASONING_BODY = ( + "reasoning_body" # OpenRouter: extra_body.reasoning{enabled|effort} + ) + NONE = "none" # non-reasoning model: send nothing + + +class AudioShape(str, Enum): + FILE_BLOCK = ( + "file_block" # Interfaze: {"type":"file","file":{"file_data": data-uri}} + ) + AUDIO_URL = ( + "audio_url" # Fireworks: {"type":"audio_url","audio_url":{"url": data-uri}} + ) + INPUT_AUDIO = ( + "input_audio" # OpenRouter: {"type":"input_audio","input_audio":{data,format}} + ) + GEMINI_PART = "gemini_part" # Gemini: types.Part.from_bytes(mime_type="audio/...") + NONE = "none" + + +class ImageShape(str, Enum): + IMAGE_URL = ( + "image_url" # OpenAI-family: {"type":"image_url","image_url":{"url": data-uri}} + ) + ANTHROPIC_SOURCE = "anthropic_source" # {"type":"image","source":{base64}} + GEMINI_PART = "gemini_part" # types.Part.from_bytes(mime_type="image/...") + NONE = "none" + + +class ResponseShape(str, Enum): + OPENAI_CHAT = "openai_chat" # choices[0].message.content + ANTHROPIC_BLOCKS = "anthropic_blocks" # content[] blocks, join type=="text" + GEMINI_TEXT = "gemini_text" # response.text + + +def deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> dict: + """Recursively merge `override` onto `base`, returning a new dict. + + Nested dicts merge key-by-key; every other value (scalar, list) is replaced + wholesale by `override`. Neither input is mutated — the resolution chain + (adapter defaults ← target ← CLI) reuses the same base dict repeatedly. + """ + result = copy.deepcopy(dict(base)) + for key, val in override.items(): + existing = result.get(key) + if isinstance(existing, dict) and isinstance(val, Mapping): + result[key] = deep_merge(existing, val) + else: + result[key] = copy.deepcopy(val) + return result + + +@dataclass(frozen=True) +class ReasoningCap: + style: ReasoningStyle + # NB: keys are `off_value`/`on_value`, NOT `off`/`on` — YAML 1.1 parses bare + # `off:`/`on:` as booleans, so `off: low` would become `{False: "low"}`. + off_value: Any = None # concrete value sent for the "off" mode (may be a FLOOR) + on_value: Any = None # concrete value sent for the "high"/"on" mode + true_off: bool = True # is `off` a real disable, or does it still think? + temperature_when_on: bool = True # may temperature accompany active thinking? + extra: dict = field( + default_factory=dict + ) # provider pins, budget/max coupling, etc. + + +@dataclass(frozen=True) +class MediaCap: + audio: AudioShape = AudioShape.NONE + image: ImageShape = ImageShape.NONE + + +@dataclass(frozen=True) +class Capabilities: + reasoning: ReasoningCap + media: MediaCap + response: ResponseShape + max_tokens_param: str = "max_tokens" + + @classmethod + def from_dict(cls, d: Mapping[str, Any]) -> Capabilities: + r = dict(d.get("reasoning") or {}) + try: + style = ReasoningStyle(r.get("style", "none")) + except ValueError as exc: + raise ValueError( + f"unknown reasoning style {r.get('style')!r}; " + f"expected one of {[s.value for s in ReasoningStyle]}" + ) from exc + reasoning = ReasoningCap( + style=style, + off_value=r.get("off_value"), + on_value=r.get("on_value"), + true_off=bool(r.get("true_off", True)), + temperature_when_on=bool(r.get("temperature_when_on", True)), + extra=dict(r.get("extra") or {}), + ) + + m = dict(d.get("media") or {}) + media = MediaCap( + audio=AudioShape(m.get("audio", "none")), + image=ImageShape(m.get("image", "none")), + ) + + return cls( + reasoning=reasoning, + media=media, + response=ResponseShape(d.get("response", "openai_chat")), + max_tokens_param=d.get("max_tokens_param", "max_tokens"), + ) diff --git a/bench_core/config.py b/bench_core/config.py new file mode 100644 index 0000000..7ed027d --- /dev/null +++ b/bench_core/config.py @@ -0,0 +1,196 @@ +"""Targets + provider registry + the capability merge chain. + +A provider ships defaults (base_url, key precedence, and the host's baseline +capabilities); a target YAML declares only what differs; per-benchmark and CLI +overrides layer on top. `resolve_capabilities` collapses the chain into a typed +`Capabilities`; `build_adapter` maps the provider name to its adapter class. + +Adding a MODEL is a YAML file. Adding a PROVIDER is one registry entry here. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + +from bench_core.capabilities import Capabilities, deep_merge +from bench_core.providers.anthropic import AnthropicAdapter +from bench_core.providers.gemini import GeminiAdapter +from bench_core.providers.openai_compat import OpenAICompatAdapter + +# --- provider registry ---------------------------------------------------- + + +@dataclass +class ProviderSpec: + adapter_cls: type + base_url: str | None + key_spec: list[str] + capability_defaults: dict + + +_OPENAI_MEDIA = {"image": "image_url"} + +PROVIDERS: dict[str, ProviderSpec] = { + "openai": ProviderSpec( + OpenAICompatAdapter, + "https://api.openai.com/v1", + ["OPENAI_API_KEY"], + { + "reasoning": { + "style": "effort", + "off_value": "none", + "on_value": "high", + "true_off": True, + "temperature_when_on": False, + }, + "media": _OPENAI_MEDIA, + "response": "openai_chat", + }, + ), + "fireworks": ProviderSpec( + OpenAICompatAdapter, + "https://api.fireworks.ai/inference/v1", + ["FIREWORKS_API_KEY", "fireworks_api_key"], + { + # Fireworks "none" is a FLOOR (still thinks); temperature is sent regardless. + "reasoning": { + "style": "effort", + "off_value": "none", + "on_value": "high", + "true_off": False, + "temperature_when_on": True, + }, + "media": {"image": "image_url", "audio": "audio_url"}, + "response": "openai_chat", + }, + ), + "openrouter": ProviderSpec( + OpenAICompatAdapter, + "https://openrouter.ai/api/v1", + ["OPENROUTER_API_KEY", "openrouter_api_key"], + { + "reasoning": { + "style": "reasoning_body", + "off_value": False, + "on_value": True, + "true_off": True, + "extra": {"toggle": "enabled"}, + }, + "media": {"image": "image_url", "audio": "input_audio"}, + "response": "openai_chat", + }, + ), + "interfaze": ProviderSpec( + OpenAICompatAdapter, + None, + ["INTERFAZE_API_KEY"], + { + # Interfaze reasoning vocab is off/minimal/low/medium/high/on/auto (NOT "none"). + "reasoning": { + "style": "effort", + "off_value": "off", + "on_value": "high", + "true_off": True, + "temperature_when_on": True, + }, + "media": {"image": "image_url", "audio": "file_block"}, + "response": "openai_chat", + }, + ), + "anthropic": ProviderSpec( + AnthropicAdapter, + None, + ["ANTHROPIC_API_KEY"], + { + "reasoning": { + "style": "disabled_block", + "on_value": 10000, + "true_off": True, + "temperature_when_on": False, + "extra": {"off_max_tokens": 1024, "on_max_tokens": 16000}, + }, + "media": {"image": "anthropic_source"}, + "response": "anthropic_blocks", + }, + ), + "gemini": ProviderSpec( + GeminiAdapter, + None, + ["GEMINI_API_KEY", "GEMINI_KEY", "GOOGLE_API_KEY"], + { + # 3.x default; 2.5 targets override to thinking_budget. "low" floor is + # safe for 3.7+ flash and Pro (both reject "minimal"). + "reasoning": { + "style": "thinking_level", + "off_value": "low", + "on_value": "high", + "true_off": False, + }, + "media": {"image": "gemini_part", "audio": "gemini_part"}, + "response": "gemini_text", + }, + ), +} + + +# --- targets --------------------------------------------------------------- + + +@dataclass +class Target: + name: str + provider: str + model_id: str + capabilities: dict = field(default_factory=dict) + overrides: dict = field(default_factory=dict) + harness_specific: dict = field(default_factory=dict) + raw: dict = field(default_factory=dict) + + +def load_target(path: str | Path) -> Target: + data = yaml.safe_load(Path(path).read_text()) or {} + if data.get("provider") not in PROVIDERS: + raise ValueError( + f"unknown provider {data.get('provider')!r}; known: {sorted(PROVIDERS)}" + ) + return Target( + name=data["name"], + provider=data["provider"], + model_id=data["model_id"], + capabilities=data.get("capabilities") or {}, + overrides=data.get("overrides") or {}, + harness_specific=data.get("harness_specific") or {}, + raw=data, + ) + + +def resolve_capabilities( + target: Target, + benchmark: str | None = None, + cli_overrides: dict | None = None, +) -> Capabilities: + spec = PROVIDERS[target.provider] + merged: dict[str, Any] = deep_merge(spec.capability_defaults, target.capabilities) + if benchmark and benchmark in target.overrides: + merged = deep_merge(merged, target.overrides[benchmark]) + if cli_overrides: + merged = deep_merge(merged, cli_overrides) + return Capabilities.from_dict(merged) + + +def build_adapter(target: Target): + spec = PROVIDERS[target.provider] + base_url = spec.base_url + if target.provider == "interfaze": + base_url = os.getenv("OPENAI_BASE_URL", "https://api.interfaze.ai/v1") + return spec.adapter_cls( + name=target.provider, + base_url=base_url, + key_spec=list(spec.key_spec), + capability_defaults=spec.capability_defaults, + ) diff --git a/bench_core/decode.py b/bench_core/decode.py new file mode 100644 index 0000000..3af19be --- /dev/null +++ b/bench_core/decode.py @@ -0,0 +1,69 @@ +"""Normalize a host response into `Response`. One function per response shape; +the reasoning-token fallbacks (Fireworks top-level, Gemini thoughts count) live +here instead of in every runner. +""" + +from __future__ import annotations + +from typing import Any + +from bench_core.capabilities import ResponseShape +from bench_core.response import Response + + +def _get(obj: Any, name: str, default=None): + return getattr(obj, name, default) + + +def _decode_openai(raw: Any) -> Response: + choice = raw.choices[0] + usage = _get(raw, "usage") + reasoning = None + if usage is not None: + details = _get(usage, "completion_tokens_details") + if details is not None: + reasoning = _get(details, "reasoning_tokens") + if reasoning is None: # Fireworks/Inkling report it at the top level + reasoning = _get(usage, "reasoning_tokens") + return Response( + text=_get(choice.message, "content") or "", + reasoning_tokens=reasoning, + input_tokens=_get(usage, "prompt_tokens") if usage else None, + output_tokens=_get(usage, "completion_tokens") if usage else None, + finish_reason=_get(choice, "finish_reason"), + raw_id=_get(raw, "id"), + ) + + +def _decode_anthropic(raw: Any) -> Response: + text = "".join(b.text for b in (raw.content or []) if _get(b, "type") == "text") + usage = _get(raw, "usage") + return Response( + text=text, + reasoning_tokens=None, # Anthropic does not report thinking tokens separately + input_tokens=_get(usage, "input_tokens") if usage else None, + output_tokens=_get(usage, "output_tokens") if usage else None, + raw_id=_get(raw, "id"), + ) + + +def _decode_gemini(raw: Any) -> Response: + meta = _get(raw, "usage_metadata") + return Response( + text=_get(raw, "text") or "", + reasoning_tokens=_get(meta, "thoughts_token_count") if meta else None, + input_tokens=_get(meta, "prompt_token_count") if meta else None, + output_tokens=_get(meta, "candidates_token_count") if meta else None, + raw_id=_get(raw, "response_id"), + ) + + +_DECODERS = { + ResponseShape.OPENAI_CHAT: _decode_openai, + ResponseShape.ANTHROPIC_BLOCKS: _decode_anthropic, + ResponseShape.GEMINI_TEXT: _decode_gemini, +} + + +def decode_response(raw: Any, shape: ResponseShape) -> Response: + return _DECODERS[shape](raw) diff --git a/bench_core/errors.py b/bench_core/errors.py new file mode 100644 index 0000000..de10871 --- /dev/null +++ b/bench_core/errors.py @@ -0,0 +1,101 @@ +"""Classify a provider exception into the signal the fallback ladder acts on. + +Status code comes from the SDK attribute when present, else the leading digits +of the message (google-genai raises a plain exception whose text starts with +"400 INVALID_ARGUMENT ..."). PARAM_REJECTED carries an `action` the ladder maps +to a request mutation, so a genuinely new model degrades instead of crashing. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from enum import Enum + + +class ErrorKind(str, Enum): + PARAM_REJECTED = ( + "param_rejected" # a param the model won't accept -> adjust + retry + ) + RATE_LIMITED = "rate_limited" # 429 -> backoff + retry + TRANSIENT = "transient" # timeout / 5xx / connection -> backoff + retry + EMPTY_CONTENT = "empty_content" # returned nothing usable -> retry + FATAL = "fatal" # auth / missing model / unrecognized 4xx -> fail the sample + + +@dataclass +class Classification: + kind: ErrorKind + action: str | None = ( + None # raise_thinking_floor | force_reasoning_floor | drop_temperature | drop_param + ) + param: str | None = None + message: str = "" + + +# 400s that name a specific, adjustable parameter. Ordered — first match wins. +_PARAM_PATTERNS: list[tuple[re.Pattern, str, str | None]] = [ + ( + re.compile(r"thinking level \w+ is not supported", re.IGNORECASE), + "raise_thinking_floor", + "thinking_level", + ), + ( + re.compile( + r"reasoning is mandatory|reasoning.*cannot be disabled|cannot disable reasoning", + re.IGNORECASE, + ), + "force_reasoning_floor", + "reasoning", + ), + (re.compile(r"temperature", re.IGNORECASE), "drop_temperature", "temperature"), + ( + re.compile( + r"(?:unknown|unsupported|unexpected|invalid) (?:parameter|argument|keyword)[:\s]+['\"]?([\w.]+)", + re.IGNORECASE, + ), + "drop_param", + None, + ), +] + +_TRANSIENT_STATUS = {500, 502, 503, 504} +_TRANSIENT_MSG = re.compile( + r"timed out|timeout|connection|temporarily unavailable|overloaded", re.IGNORECASE +) + + +def _status_of(exc: Exception) -> int | None: + for attr in ("status_code", "code", "http_status"): + val = getattr(exc, attr, None) + if isinstance(val, int): + return val + m = re.match(r"\s*(\d{3})\b", str(exc)) + return int(m.group(1)) if m else None + + +def classify(exc: Exception) -> Classification: + msg = str(exc) + status = _status_of(exc) + + if isinstance(exc, TimeoutError) or _TRANSIENT_MSG.search(msg): + return Classification(ErrorKind.TRANSIENT, message=msg) + if status == 429: + return Classification(ErrorKind.RATE_LIMITED, message=msg) + if status in _TRANSIENT_STATUS: + return Classification(ErrorKind.TRANSIENT, message=msg) + if status in (401, 403, 404): + return Classification(ErrorKind.FATAL, message=msg) + + if status == 400 or status is None: + for pattern, action, param in _PARAM_PATTERNS: + m = pattern.search(msg) + if m: + captured = param or (m.group(1) if m.groups() else None) + return Classification( + ErrorKind.PARAM_REJECTED, action=action, param=captured, message=msg + ) + if status == 400: + return Classification(ErrorKind.FATAL, message=msg) + + return Classification(ErrorKind.FATAL, message=msg) diff --git a/bench_core/execute.py b/bench_core/execute.py new file mode 100644 index 0000000..0dde322 --- /dev/null +++ b/bench_core/execute.py @@ -0,0 +1,110 @@ +"""The fallback ladder — one request through an adapter, self-healing the +declared-capability gaps a genuinely new model exposes. + +`PARAM_REJECTED` is handled here: mutate the request/caps per the classified +action, record a capability_hint (so the user promotes the fix into the target +YAML), and retry immediately. Everything else (rate-limit, transient, empty, +fatal) is raised as `BenchError` carrying the classification, for the runner to +retry-with-backoff or fail. +""" + +from __future__ import annotations + +import dataclasses + +from bench_core.capabilities import Capabilities, ReasoningStyle +from bench_core.errors import Classification, ErrorKind +from bench_core.request import Request +from bench_core.response import Response + +_THINKING_FLOORS = ["minimal", "low", "medium", "high"] + + +class BenchError(Exception): + def __init__(self, classification: Classification): + super().__init__(classification.message or classification.kind.value) + self.classification = classification + + +def _raise_thinking_floor(caps: Capabilities) -> tuple[Capabilities, str]: + cur = caps.reasoning.off_value + try: + nxt = _THINKING_FLOORS[_THINKING_FLOORS.index(cur) + 1] + except (ValueError, IndexError): + nxt = "low" + reasoning = dataclasses.replace(caps.reasoning, off_value=nxt) + return dataclasses.replace( + caps, reasoning=reasoning + ), f"thinking floor raised to {nxt}" + + +def _force_reasoning_floor(caps: Capabilities) -> tuple[Capabilities, str]: + r = caps.reasoning + if r.style is ReasoningStyle.REASONING_BODY: + extra = dict(r.extra) + extra["toggle"] = "effort" + reasoning = dataclasses.replace( + r, extra=extra, off_value="minimal", true_off=False + ) + elif r.style is ReasoningStyle.EFFORT: + reasoning = dataclasses.replace(r, off_value="low", true_off=False) + else: + reasoning = dataclasses.replace(r, true_off=False) + return dataclasses.replace( + caps, reasoning=reasoning + ), "reasoning cannot be disabled; using floor" + + +def _apply_action( + c: Classification, req: Request, caps: Capabilities +) -> tuple[Request, Capabilities, str] | None: + """Return a mutated (req, caps, hint), or None if the action can't be + applied here (escalates to fatal).""" + if c.action == "raise_thinking_floor": + caps, hint = _raise_thinking_floor(caps) + return req, caps, hint + if c.action == "force_reasoning_floor": + caps, hint = _force_reasoning_floor(caps) + return req, caps, hint + if c.action == "drop_temperature": + req = dataclasses.replace(req, temperature=None) + return req, caps, "temperature dropped (rejected with reasoning)" + return None + + +def execute( + adapter, + client, + req: Request, + caps: Capabilities, + model_id: str, + *, + max_param_retries: int = 4, +) -> tuple[Response, list[str]]: + hints: list[str] = [] + work_req, work_caps = req, caps + + for _ in range(max_param_retries + 1): + encoded = adapter.encode(work_req, work_caps, model_id) + try: + raw = adapter.call(client, encoded) + except Exception as exc: + c = adapter.classify_error(exc) + if c.kind is ErrorKind.PARAM_REJECTED: + applied = _apply_action(c, work_req, work_caps) + if applied is not None: + work_req, work_caps, hint = applied + hints.append(hint) + continue + raise BenchError(c) from exc + + resp = adapter.decode(raw, work_caps) + if not resp.text.strip(): + raise BenchError( + Classification(ErrorKind.EMPTY_CONTENT, message="empty content") + ) + return resp, hints + + raise BenchError( + Classification(ErrorKind.FATAL, message="parameter adjustments exhausted") + ) diff --git a/bench_core/media.py b/bench_core/media.py new file mode 100644 index 0000000..3b326d4 --- /dev/null +++ b/bench_core/media.py @@ -0,0 +1,112 @@ +"""Media block builders — one per host shape, so the benchmark never picks a +shape and the silent-drop look-alikes (audio_url vs input_audio) can't be +confused. `encode_image` is the single place RGB-convert + resize + encode +happens (the audit found it in only 1 of 9 OCRBench variants). +""" + +from __future__ import annotations + +import base64 +import io +from dataclasses import dataclass +from typing import Any + +from bench_core.capabilities import AudioShape, ImageShape +from bench_core.request import AudioPart, ImagePart + + +@dataclass(frozen=True) +class GeminiPart: + """Marker for a native `google.genai` part. Kept SDK-free here so media + logic is testable without the SDK; the Gemini adapter maps it to + `types.Part.from_bytes(data=..., mime_type=...)`.""" + + data: bytes + mime: str + + +def _b64(data: bytes) -> str: + return base64.b64encode(data).decode("ascii") + + +def _data_uri(data: bytes, mime: str) -> str: + return f"data:{mime};base64,{_b64(data)}" + + +def _subtype(mime: str) -> str: + """ "audio/wav" -> "wav".""" + return mime.split("/", 1)[-1] + + +def audio_block(part: AudioPart, shape: AudioShape) -> Any: + if shape is AudioShape.FILE_BLOCK: + return { + "type": "file", + "file": { + "filename": f"audio.{_subtype(part.mime)}", + "file_data": _data_uri(part.data, part.mime), + }, + } + if shape is AudioShape.AUDIO_URL: + return { + "type": "audio_url", + "audio_url": {"url": _data_uri(part.data, part.mime)}, + } + if shape is AudioShape.INPUT_AUDIO: + # Raw base64 (NOT a data-URI) + explicit format. Sending the audio_url + # data-URI here is accepted but silently drops the audio. + return { + "type": "input_audio", + "input_audio": {"data": _b64(part.data), "format": _subtype(part.mime)}, + } + if shape is AudioShape.GEMINI_PART: + return GeminiPart(data=part.data, mime=part.mime) + raise ValueError(f"cannot send audio to a model with audio shape {shape!r}") + + +def image_block(part: ImagePart, shape: ImageShape) -> Any: + if shape is ImageShape.IMAGE_URL: + return { + "type": "image_url", + "image_url": {"url": _data_uri(part.data, part.mime)}, + } + if shape is ImageShape.ANTHROPIC_SOURCE: + return { + "type": "image", + "source": { + "type": "base64", + "media_type": part.mime, + "data": _b64(part.data), + }, + } + if shape is ImageShape.GEMINI_PART: + return GeminiPart(data=part.data, mime=part.mime) + raise ValueError(f"cannot send image to a model with image shape {shape!r}") + + +def encode_image( + pil_image, mime: str = "image/jpeg", max_side: int | None = None +) -> ImagePart: + """PIL image -> ImagePart, doing the conversions every consumer needs once: + RGB-convert for JPEG (JPEG can't hold alpha; the omission crashed 8/9 OCR + variants on RGBA/P images), optional longest-side downscale, then encode.""" + from PIL import Image + + img = pil_image + if max_side is not None: + w, h = img.size + longest = max(w, h) + if longest > max_side: + scale = max_side / longest + img = img.resize((round(w * scale), round(h * scale)), Image.LANCZOS) + + fmt = _subtype(mime).upper() + if fmt in ("JPG", "JPEG"): + fmt = "JPEG" + if img.mode != "RGB": + img = img.convert("RGB") + + buf = io.BytesIO() + save_kwargs = {"quality": 95} if fmt == "JPEG" else {} + img.save(buf, format=fmt, **save_kwargs) + return ImagePart(data=buf.getvalue(), mime=mime) diff --git a/bench_core/providers/__init__.py b/bench_core/providers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bench_core/providers/anthropic.py b/bench_core/providers/anthropic.py new file mode 100644 index 0000000..71775ae --- /dev/null +++ b/bench_core/providers/anthropic.py @@ -0,0 +1,82 @@ +"""Anthropic adapter. System messages become the top-level `system` param; +`max_tokens` is mandatory and must exceed `budget_tokens` when thinking is +enabled — both handled declaratively from the cap. +""" + +from __future__ import annotations + +from bench_core.capabilities import Capabilities +from bench_core.media import image_block +from bench_core.providers.base import ProviderAdapter +from bench_core.reasoning import build_reasoning +from bench_core.request import ImagePart, Message, Request, TextPart + +_DEFAULT_OFF_MAX = 1024 +_DEFAULT_ON_MAX = 16000 +_BUDGET_MARGIN = 4096 + + +class AnthropicAdapter(ProviderAdapter): + def __init__(self, name, base_url, key_spec, capability_defaults): + self.name = name + self.base_url = base_url + self.key_spec = key_spec + self.capability_defaults = capability_defaults + + def build_client(self, api_key: str | None = None): + from anthropic import Anthropic + + return Anthropic( + api_key=api_key or self.resolve_key(), timeout=180.0, max_retries=2 + ) + + def _content(self, msg: Message, caps: Capabilities): + if all(isinstance(p, TextPart) for p in msg.parts): + return "".join(p.text for p in msg.parts) + blocks = [] + for p in msg.parts: + if isinstance(p, TextPart): + blocks.append({"type": "text", "text": p.text}) + elif isinstance(p, ImagePart): + blocks.append(image_block(p, caps.media.image)) + return blocks + + def encode(self, req: Request, caps: Capabilities, model_id: str) -> dict: + system = "".join( + p.text + for m in req.messages + if m.role == "system" + for p in m.parts + if isinstance(p, TextPart) + ) + messages = [ + {"role": m.role, "content": self._content(m, caps)} + for m in req.messages + if m.role != "system" + ] + kwargs: dict = {"model": model_id, "messages": messages} + if system: + kwargs["system"] = system + + inj = build_reasoning(req.reasoning.mode, caps.reasoning) + extra = caps.reasoning.extra + if inj.anthropic_thinking is not None: + kwargs["thinking"] = inj.anthropic_thinking + + thinking_on = (inj.anthropic_thinking or {}).get("type") == "enabled" + if thinking_on: + budget = inj.anthropic_thinking["budget_tokens"] + max_tokens = req.max_tokens or extra.get("on_max_tokens", _DEFAULT_ON_MAX) + if max_tokens <= budget: + max_tokens = budget + _BUDGET_MARGIN + else: + max_tokens = req.max_tokens or extra.get("off_max_tokens", _DEFAULT_OFF_MAX) + kwargs["max_tokens"] = max_tokens + + if req.temperature is not None and inj.allow_temperature: + kwargs["temperature"] = req.temperature + + return kwargs + + def call(self, client, encoded: dict): + return client.messages.create(**encoded) diff --git a/bench_core/providers/base.py b/bench_core/providers/base.py new file mode 100644 index 0000000..370e5e4 --- /dev/null +++ b/bench_core/providers/base.py @@ -0,0 +1,51 @@ +"""Provider adapter contract + shared behavior. + +`decode` and `classify_error` are identical across hosts (they dispatch on the +capability's response shape / the exception), so they live here. Subclasses +supply `build_client`, `encode`, and `call` — the parts that are genuinely +provider-specific. +""" + +from __future__ import annotations + +import os +from typing import Any + +from bench_core.capabilities import Capabilities +from bench_core.decode import decode_response +from bench_core.errors import Classification, classify +from bench_core.request import Request +from bench_core.response import Response + + +class ProviderAdapter: + name: str + base_url: str | None + key_spec: list[str] # env-var precedence, incl. lowercase fallbacks + capability_defaults: dict + + def resolve_key(self) -> str | None: + for env in self.key_spec: + val = os.getenv(env) + if val: + return val + return None + + # --- provider-specific (override) --- + def build_client(self, api_key: str | None = None) -> Any: # pragma: no cover + raise NotImplementedError + + def encode( + self, req: Request, caps: Capabilities, model_id: str + ) -> Any: # pragma: no cover + raise NotImplementedError + + def call(self, client: Any, encoded: Any) -> Any: # pragma: no cover + raise NotImplementedError + + # --- shared --- + def decode(self, raw: Any, caps: Capabilities) -> Response: + return decode_response(raw, caps.response) + + def classify_error(self, exc: Exception) -> Classification: + return classify(exc) diff --git a/bench_core/providers/gemini.py b/bench_core/providers/gemini.py new file mode 100644 index 0000000..f5fcb17 --- /dev/null +++ b/bench_core/providers/gemini.py @@ -0,0 +1,72 @@ +"""Gemini (native google-genai) adapter. Reasoning becomes a ThinkingConfig +(level for 3.x, budget for 2.5); media parts become genai Parts. The client is +cached at the factory level to avoid the "client has been closed" GC bug. +""" + +from __future__ import annotations + +from bench_core.capabilities import Capabilities +from bench_core.media import GeminiPart, audio_block, image_block +from bench_core.providers.base import ProviderAdapter +from bench_core.reasoning import build_reasoning +from bench_core.request import AudioPart, ImagePart, Request, TextPart + + +class GeminiAdapter(ProviderAdapter): + def __init__(self, name, base_url, key_spec, capability_defaults): + self.name = name + self.base_url = base_url + self.key_spec = key_spec + self.capability_defaults = capability_defaults + + def build_client(self, api_key: str | None = None): + from google import genai + + return genai.Client(api_key=api_key or self.resolve_key()) + + def _contents(self, req: Request, caps: Capabilities) -> list: + from google.genai import types + + contents: list = [] + for m in req.messages: + for p in m.parts: + if isinstance(p, TextPart): + contents.append(p.text) + elif isinstance(p, ImagePart): + gp = image_block(p, caps.media.image) + contents.append(self._to_part(gp, types)) + elif isinstance(p, AudioPart): + gp = audio_block(p, caps.media.audio) + contents.append(self._to_part(gp, types)) + return contents + + @staticmethod + def _to_part(gp: GeminiPart, types): + return types.Part.from_bytes(data=gp.data, mime_type=gp.mime) + + def encode(self, req: Request, caps: Capabilities, model_id: str) -> dict: + from google.genai import types + + inj = build_reasoning(req.reasoning.mode, caps.reasoning) + thinking = None + if inj.thinking_level is not None: + thinking = types.ThinkingConfig(thinking_level=inj.thinking_level) + elif inj.thinking_budget is not None: + thinking = types.ThinkingConfig(thinking_budget=inj.thinking_budget) + + config_kwargs: dict = {} + if thinking is not None: + config_kwargs["thinking_config"] = thinking + if req.temperature is not None and inj.allow_temperature: + config_kwargs["temperature"] = req.temperature + if req.max_tokens is not None: + config_kwargs["max_output_tokens"] = req.max_tokens + + return { + "model": model_id, + "contents": self._contents(req, caps), + "config": types.GenerateContentConfig(**config_kwargs), + } + + def call(self, client, encoded: dict): + return client.models.generate_content(**encoded) diff --git a/bench_core/providers/openai_compat.py b/bench_core/providers/openai_compat.py new file mode 100644 index 0000000..73c7c17 --- /dev/null +++ b/bench_core/providers/openai_compat.py @@ -0,0 +1,70 @@ +"""Adapter for hosts that speak OpenAI Chat Completions: openai, fireworks, +openrouter, interfaze. One `encode`; the four concrete providers differ only in +base_url / key_spec / capability defaults (defined in their target YAML + a thin +factory), not in code. +""" + +from __future__ import annotations + +from typing import Any + +from bench_core.capabilities import Capabilities +from bench_core.media import audio_block, image_block +from bench_core.providers.base import ProviderAdapter +from bench_core.reasoning import build_reasoning +from bench_core.request import AudioPart, ImagePart, Message, Request, TextPart + + +class OpenAICompatAdapter(ProviderAdapter): + def __init__(self, name, base_url, key_spec, capability_defaults): + self.name = name + self.base_url = base_url + self.key_spec = key_spec + self.capability_defaults = capability_defaults + + def build_client(self, api_key: str | None = None): + from openai import OpenAI + + return OpenAI( + api_key=api_key or self.resolve_key(), + base_url=self.base_url, + timeout=180.0, + max_retries=2, + ) + + def _content(self, msg: Message, caps: Capabilities) -> Any: + if all(isinstance(p, TextPart) for p in msg.parts): + return "".join(p.text for p in msg.parts) + blocks = [] + for p in msg.parts: + if isinstance(p, TextPart): + blocks.append({"type": "text", "text": p.text}) + elif isinstance(p, ImagePart): + blocks.append(image_block(p, caps.media.image)) + elif isinstance(p, AudioPart): + blocks.append(audio_block(p, caps.media.audio)) + return blocks + + def encode(self, req: Request, caps: Capabilities, model_id: str) -> dict: + kwargs: dict = { + "model": model_id, + "messages": [ + {"role": m.role, "content": self._content(m, caps)} + for m in req.messages + ], + } + + inj = build_reasoning(req.reasoning.mode, caps.reasoning) + kwargs.update(inj.kwargs) + if inj.extra_body: + kwargs["extra_body"] = inj.extra_body + + if req.temperature is not None and inj.allow_temperature: + kwargs["temperature"] = req.temperature + if req.max_tokens is not None: + kwargs[caps.max_tokens_param] = req.max_tokens + + return kwargs + + def call(self, client, encoded: dict): + return client.chat.completions.create(**encoded) diff --git a/bench_core/reasoning.py b/bench_core/reasoning.py new file mode 100644 index 0000000..791f643 --- /dev/null +++ b/bench_core/reasoning.py @@ -0,0 +1,109 @@ +"""Translate an abstract reasoning *mode* into the concrete directive a host +expects — the one place the audit's six divergent Gemini-floor encodings (plus +the OpenAI/Fireworks/Anthropic/OpenRouter variants) now live. + +A benchmark asks for a mode ("off" / "low" / "medium" / "high"); the model's +declared `ReasoningCap` says how that host expresses it and what its floor is. +`build_reasoning` returns a `ReasoningInjection` describing where the directive +goes; each provider adapter reads only the field for its mechanism. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from bench_core.capabilities import ReasoningCap, ReasoningStyle + +_HIGH_MODES = {"high", "on", "max"} +_OFF_MODES = {"off", "none", "disabled"} + + +@dataclass +class ReasoningInjection: + """How to attach reasoning to a request. A given cap fills exactly one of + the mechanism fields; the rest stay empty/None so an adapter that reads the + wrong field simply sees "nothing to do".""" + + kwargs: dict = field( + default_factory=dict + ) # top-level create() kwargs (reasoning_effort) + extra_body: dict = field( + default_factory=dict + ) # OpenRouter reasoning + provider pin + thinking_level: str | None = None # Gemini 3.x + thinking_budget: int | None = None # Gemini 2.5 + anthropic_thinking: dict | None = ( + None # {"type":"disabled"} | {"type":"enabled",...} + ) + allow_temperature: bool = True + + +def _resolve_value(mode: str, cap: ReasoningCap) -> Any: + """Concrete value for a mode: `off`/`on` come from the cap; an intermediate + mode passes through literally (caller owns vocabulary validity).""" + if mode in _OFF_MODES: + return cap.off_value + if mode in _HIGH_MODES: + return cap.on_value + return mode + + +def _allow_temperature(mode: str, cap: ReasoningCap) -> bool: + """Temperature is allowed when thinking is not engaged, or when the host + tolerates temperature alongside active thinking. An off run that truly + disables thinking counts as "not engaged".""" + if mode in _OFF_MODES and cap.true_off: + return True + return cap.temperature_when_on + + +def build_reasoning(mode: str, cap: ReasoningCap) -> ReasoningInjection: + style = cap.style + allow_temp = _allow_temperature(mode, cap) + + if style is ReasoningStyle.NONE: + return ReasoningInjection(allow_temperature=allow_temp) + + if style is ReasoningStyle.EFFORT: + return ReasoningInjection( + kwargs={"reasoning_effort": _resolve_value(mode, cap)}, + allow_temperature=allow_temp, + ) + + if style is ReasoningStyle.THINKING_LEVEL: + return ReasoningInjection( + thinking_level=_resolve_value(mode, cap), + allow_temperature=allow_temp, + ) + + if style is ReasoningStyle.THINKING_BUDGET: + return ReasoningInjection( + thinking_budget=_resolve_value(mode, cap), + allow_temperature=allow_temp, + ) + + if style is ReasoningStyle.DISABLED_BLOCK: + if mode in _OFF_MODES: + thinking = {"type": "disabled"} + else: + thinking = {"type": "enabled", "budget_tokens": cap.on_value} + return ReasoningInjection( + anthropic_thinking=thinking, + allow_temperature=allow_temp, + ) + + if style is ReasoningStyle.REASONING_BODY: + toggle = cap.extra.get("toggle", "enabled") + value = _resolve_value(mode, cap) + if toggle == "effort": + reasoning = {"effort": value} + else: # "enabled" boolean toggle + reasoning = {"enabled": bool(value)} + extra_body: dict = {"reasoning": reasoning} + pin = cap.extra.get("provider_only") + if pin: + extra_body["provider"] = {"only": list(pin)} + return ReasoningInjection(extra_body=extra_body, allow_temperature=allow_temp) + + raise ValueError(f"unhandled reasoning style {style!r}") diff --git a/bench_core/request.py b/bench_core/request.py new file mode 100644 index 0000000..26a9cbe --- /dev/null +++ b/bench_core/request.py @@ -0,0 +1,53 @@ +"""Provider-agnostic request the benchmark layer builds. Benchmarks construct +these and never name a provider; the adapter's `encode` turns them into the +host-native payload. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class TextPart: + text: str + + +@dataclass +class ImagePart: + """Already-encoded image bytes + its mime (e.g. "image/jpeg" for OCRBench, + "image/png" for olmOCR — a *benchmark* choice; the wire *shape* is the + adapter's job). Build via `media.encode_image` so RGB-convert/resize happen + once, centrally, instead of in 1-of-9 variants.""" + + data: bytes + mime: str = "image/jpeg" + + +@dataclass +class AudioPart: + data: bytes + mime: str = "audio/wav" + + +Part = TextPart | ImagePart | AudioPart + + +@dataclass +class Message: + role: str # "system" | "user" | "assistant" + parts: list[Part] = field(default_factory=list) + + +@dataclass +class ReasoningSpec: + mode: str = "off" # off | low | medium | high (host floor applied by the cap) + + +@dataclass +class Request: + messages: list[Message] + reasoning: ReasoningSpec = field(default_factory=ReasoningSpec) + temperature: float | None = 0.0 + max_tokens: int | None = None + schema: dict | None = None # optional structured-output JSON schema diff --git a/bench_core/response.py b/bench_core/response.py new file mode 100644 index 0000000..d691f7e --- /dev/null +++ b/bench_core/response.py @@ -0,0 +1,17 @@ +"""Normalized response — one shape regardless of host, so benchmark `parse` +never touches choices[0] / content-blocks / .text directly. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass +class Response: + text: str + reasoning_tokens: int | None = None + input_tokens: int | None = None + output_tokens: int | None = None + finish_reason: str | None = None + raw_id: str | None = None diff --git a/bench_core/results.py b/bench_core/results.py new file mode 100644 index 0000000..4df36a2 --- /dev/null +++ b/bench_core/results.py @@ -0,0 +1,99 @@ +"""The single result contract. One slug, one directory layout, one discovery +path — so a run can't be silently dropped by a prefix mismatch and two models +can't overwrite each other's checkpoint. + + results/// + responses.jsonl # resume source, one row per sample + metrics.json # canonical schema (see cli/benchmark code) + run.json # config snapshot +""" + +from __future__ import annotations + +import json +import os +import re +from collections.abc import Callable, Iterable +from pathlib import Path + +DEFAULT_ROOT = Path("results") + + +def model_slug(name: str) -> str: + """The ONE slug rule. Strip any provider path, then collapse to + lowercase-dash — so fireworks/openrouter/google spellings of the same model + resolve to the same directory.""" + leaf = name.rsplit("/", 1)[-1] + # Preserve dots/underscores/dashes (filesystem-safe and keeps version numbers + # like "3.7" intact) so the slug matches the target's YAML `name` and no + # un-mangling step is ever needed. Only collapse genuinely unsafe chars. + return re.sub(r"[^a-z0-9._-]+", "-", leaf.lower()).strip("-") + + +class RunStore: + def __init__(self, benchmark: str, target: str, root: Path | str = DEFAULT_ROOT): + self.benchmark = benchmark + self.target = target + self.dir = Path(root) / benchmark / model_slug(target) + + @property + def responses_path(self) -> Path: + return self.dir / "responses.jsonl" + + @property + def metrics_path(self) -> Path: + return self.dir / "metrics.json" + + @property + def run_path(self) -> Path: + return self.dir / "run.json" + + def _ensure_dir(self): + self.dir.mkdir(parents=True, exist_ok=True) + + def append_response(self, record: dict) -> None: + """Append + fsync per row so a crash mid-run loses nothing already + billed (the audit found several runners writing only at the end).""" + self._ensure_dir() + with open(self.responses_path, "a", encoding="utf-8") as f: + f.write(json.dumps(record, ensure_ascii=False) + "\n") + f.flush() + os.fsync(f.fileno()) + + def load_responses(self) -> list[dict]: + if not self.responses_path.exists(): + return [] + out = [] + with open(self.responses_path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + out.append(json.loads(line)) + return out + + def completed_ids(self, id_key: str, done: Callable[[dict], bool]) -> set: + return {r[id_key] for r in self.load_responses() if done(r)} + + def write_metrics(self, metrics: dict) -> None: + self._ensure_dir() + self.metrics_path.write_text(json.dumps(metrics, indent=2, ensure_ascii=False)) + + def write_run(self, run: dict) -> None: + self._ensure_dir() + self.run_path.write_text(json.dumps(run, indent=2, ensure_ascii=False)) + + +def discover(root: Path | str = DEFAULT_ROOT) -> list[dict]: + """Every metrics.json under the layout — content-based, the single source + the report reads.""" + root = Path(root) + out = [] + for path in sorted(root.glob("*/*/metrics.json")): + data = json.loads(path.read_text()) + data["_path"] = str(path) + out.append(data) + return out + + +def load_ids(records: Iterable[dict], id_key: str) -> set: + return {r[id_key] for r in records if id_key in r} diff --git a/bench_core/runner.py b/bench_core/runner.py new file mode 100644 index 0000000..61d1d16 --- /dev/null +++ b/bench_core/runner.py @@ -0,0 +1,137 @@ +"""The one execution harness. Owns resume, bounded concurrency, rate limiting, +retry-with-backoff, and incremental checkpointing so no benchmark reimplements +them (the audit found ~15 divergent copies). Benchmarks supply only +`build_request` and `parse`; inference goes through the fallback ladder. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field + +from bench_core.errors import ErrorKind +from bench_core.execute import BenchError, execute +from bench_core.results import RunStore + +_RETRYABLE = {ErrorKind.RATE_LIMITED, ErrorKind.TRANSIENT, ErrorKind.EMPTY_CONTENT} + + +@dataclass +class RunResult: + n_completed: int = 0 + n_failed: int = 0 + hints: list[str] = field(default_factory=list) + + +class _RateLimiter: + """Token bucket bounding how many requests *start* per second.""" + + def __init__(self, rate: float): + self.rate = rate + self.tokens = float(rate) + self.last = None + self._lock = asyncio.Lock() + + async def acquire(self): + if self.rate <= 0: + return + while True: + async with self._lock: + now = asyncio.get_running_loop().time() + if self.last is None: + self.last = now + self.tokens = min( + self.rate, self.tokens + (now - self.last) * self.rate + ) + self.last = now + if self.tokens >= 1: + self.tokens -= 1 + return + await asyncio.sleep(1.0 / self.rate) + + +async def run_benchmark( + *, + adapter, + client, + caps, + model_id: str, + samples, + build_request, + parse, + store: RunStore, + id_key: str = "id", + done=None, + rate_limit: float = 25.0, + max_in_flight: int = 8, + max_retries: int = 3, + backoff_base: float = 1.0, +) -> RunResult: + if done is None: + + def done(r): + return r.get("response") is not None + + completed = store.completed_ids(id_key, done) + pending = [s for s in samples if s[id_key] not in completed] + + limiter = _RateLimiter(rate_limit) + sem = asyncio.Semaphore(max_in_flight) + result = RunResult() + lock = asyncio.Lock() + + async def process(sample): + req = build_request(sample) + for attempt in range(max_retries + 1): + await limiter.acquire() + async with sem: + try: + resp, hints = await asyncio.to_thread( + execute, adapter, client, req, caps, model_id + ) + except BenchError as be: + kind = be.classification.kind + if kind in _RETRYABLE and attempt < max_retries: + if backoff_base > 0: + await asyncio.sleep(backoff_base * (2**attempt)) + continue + await _record_failure(sample, be, store, result, lock, id_key) + return + await _record_success( + sample, resp, hints, parse, store, result, lock, id_key + ) + return + + await asyncio.gather(*(process(s) for s in pending)) + return result + + +async def _record_success(sample, resp, hints, parse, store, result, lock, id_key): + prediction = parse(resp, sample) + record = { + id_key: sample[id_key], + "response": resp.text, + "prediction": prediction, + "reasoning_tokens": resp.reasoning_tokens, + } + if hints: + record["capability_hints"] = hints + async with lock: + store.append_response(record) + result.n_completed += 1 + for h in hints: + if h not in result.hints: + result.hints.append(h) + + +async def _record_failure(sample, be, store, result, lock, id_key): + record = { + id_key: sample[id_key], + "response": None, + "prediction": None, + "error": be.classification.kind.value, + "error_message": be.classification.message, + } + async with lock: + store.append_response(record) + result.n_failed += 1 diff --git a/bench_core/targets/claude-sonnet-4-6.yaml b/bench_core/targets/claude-sonnet-4-6.yaml new file mode 100644 index 0000000..72e6f9c --- /dev/null +++ b/bench_core/targets/claude-sonnet-4-6.yaml @@ -0,0 +1,5 @@ +name: claude-sonnet-4-6 +provider: anthropic +model_id: claude-sonnet-4-6 +# Anthropic defaults fit: thinking disabled/enabled block, temperature omitted +# when thinking is on, max_tokens forced above budget_tokens. diff --git a/bench_core/targets/gemini-2.5-flash.yaml b/bench_core/targets/gemini-2.5-flash.yaml new file mode 100644 index 0000000..117815b --- /dev/null +++ b/bench_core/targets/gemini-2.5-flash.yaml @@ -0,0 +1,11 @@ +name: gemini-2.5-flash +provider: gemini +model_id: gemini-2.5-flash +capabilities: + # 2.5 uses the older integer thinking_budget, not thinking_level. + # 0 is a genuine disable on 2.5-flash; -1 is dynamic (full thinking). + reasoning: + style: thinking_budget + off_value: 0 + on_value: -1 + true_off: true diff --git a/bench_core/targets/gemini-3.7-flash.yaml b/bench_core/targets/gemini-3.7-flash.yaml new file mode 100644 index 0000000..b66425f --- /dev/null +++ b/bench_core/targets/gemini-3.7-flash.yaml @@ -0,0 +1,5 @@ +name: gemini-3.7-flash +provider: gemini +model_id: gemini-3.7-flash +# Gemini defaults fit: thinking_level with "low" as the floor (3.7 flash rejects +# "minimal"), image/audio via native Parts. diff --git a/bench_core/targets/gpt-5.5.yaml b/bench_core/targets/gpt-5.5.yaml new file mode 100644 index 0000000..8798d13 --- /dev/null +++ b/bench_core/targets/gpt-5.5.yaml @@ -0,0 +1,5 @@ +name: gpt-5.5 +provider: openai +model_id: gpt-5.5 +# OpenAI defaults fit: reasoning_effort with "none" a true off; temperature is +# omitted automatically when reasoning is engaged. diff --git a/bench_core/targets/grok-4.3.yaml b/bench_core/targets/grok-4.3.yaml new file mode 100644 index 0000000..7720fc0 --- /dev/null +++ b/bench_core/targets/grok-4.3.yaml @@ -0,0 +1,11 @@ +name: grok-4.3 +provider: openrouter +model_id: x-ai/grok-4.3 +capabilities: + # Grok's OpenRouter endpoint rejects reasoning.enabled=false ("Reasoning is + # mandatory"), so use the effort toggle with "minimal" as the floor. + reasoning: + extra: { toggle: effort } + off_value: minimal + on_value: high + true_off: false diff --git a/bench_core/targets/inkling-small.yaml b/bench_core/targets/inkling-small.yaml new file mode 100644 index 0000000..4fbff65 --- /dev/null +++ b/bench_core/targets/inkling-small.yaml @@ -0,0 +1,10 @@ +name: inkling-small +provider: openrouter +model_id: thinkingmachines/inkling-small +capabilities: + # OpenRouter "none" truly disables reasoning here (unlike the Fireworks floor). + reasoning: + extra: { toggle: effort } + off_value: none + on_value: high + true_off: true diff --git a/bench_core/targets/inkling.yaml b/bench_core/targets/inkling.yaml new file mode 100644 index 0000000..21f8e17 --- /dev/null +++ b/bench_core/targets/inkling.yaml @@ -0,0 +1,9 @@ +# A benchmark TARGET: everything needed to run any benchmark against one model. +# Declare only what differs from the provider's defaults (see bench_core/config.py +# PROVIDERS). Fireworks defaults already encode Inkling's traits: reasoning +# effort with "none" as a FLOOR (still thinks), audio via audio_url, temperature +# sent regardless. So nothing else is needed here. +name: inkling +provider: fireworks +model_id: accounts/fireworks/models/inkling +# ci_regression: true # opt into the nightly regression matrix diff --git a/bench_core/targets/interfaze-beta.yaml b/bench_core/targets/interfaze-beta.yaml new file mode 100644 index 0000000..d3b47bc --- /dev/null +++ b/bench_core/targets/interfaze-beta.yaml @@ -0,0 +1,5 @@ +name: interfaze-beta +provider: interfaze +model_id: interfaze-beta +# Interfaze defaults fit: reasoning_effort vocab is off/minimal/low/.../high/on/auto +# (NOT "none"); audio via the OpenAI "file" block; base_url from OPENAI_BASE_URL. diff --git a/bench_core/targets/kimi-k2.6.yaml b/bench_core/targets/kimi-k2.6.yaml new file mode 100644 index 0000000..c58fe66 --- /dev/null +++ b/bench_core/targets/kimi-k2.6.yaml @@ -0,0 +1,13 @@ +name: kimi-k2.6 +provider: openrouter +model_id: moonshotai/kimi-k2.6 +capabilities: + # Pin to Moonshot's own inference (avoid reseller deployments) and disable + # reasoning via the enabled toggle. + reasoning: + extra: + toggle: enabled + provider_only: [moonshotai] + off_value: false + on_value: true + true_off: true diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_capabilities.py b/tests/test_capabilities.py new file mode 100644 index 0000000..7ae4d62 --- /dev/null +++ b/tests/test_capabilities.py @@ -0,0 +1,74 @@ +"""Capability merge + typed parsing. + +The capability layer is the heart of the redesign: provider adapters ship +default capability dicts, target YAML files override fields, and the CLI can +override again. All three collapse through `deep_merge`, then `Capabilities` +gives the adapters a typed, validated view. +""" + +from bench_core.capabilities import ( + AudioShape, + Capabilities, + ImageShape, + ReasoningStyle, + ResponseShape, + deep_merge, +) + + +def test_deep_merge_overrides_nested_scalar_keeping_siblings(): + base = {"reasoning": {"style": "effort", "off_value": "none", "true_off": True}} + override = {"reasoning": {"off_value": "low"}} + assert deep_merge(base, override) == { + "reasoning": {"style": "effort", "off_value": "low", "true_off": True} + } + + +def test_deep_merge_does_not_mutate_inputs(): + base = {"reasoning": {"off_value": "none"}} + override = {"reasoning": {"off_value": "low"}} + deep_merge(base, override) + assert base == {"reasoning": {"off_value": "none"}} # base untouched + + +def test_deep_merge_adds_new_nested_key(): + base = {"media": {"image": "image_url"}} + override = {"media": {"audio": "input_audio"}} + assert deep_merge(base, override) == { + "media": {"image": "image_url", "audio": "input_audio"} + } + + +def test_capabilities_from_dict_coerces_enums(): + caps = Capabilities.from_dict( + { + "reasoning": { + "style": "thinking_level", + "off_value": "low", + "on_value": "high", + "true_off": False, + "temperature_when_on": True, + }, + "media": {"audio": "gemini_part", "image": "gemini_part"}, + "response": "gemini_text", + "max_tokens_param": "max_tokens", + } + ) + assert caps.reasoning.style is ReasoningStyle.THINKING_LEVEL + assert caps.reasoning.true_off is False + assert caps.media.audio is AudioShape.GEMINI_PART + assert caps.media.image is ImageShape.GEMINI_PART + assert caps.response is ResponseShape.GEMINI_TEXT + + +def test_capabilities_from_dict_rejects_unknown_reasoning_style(): + import pytest + + with pytest.raises(ValueError, match="reasoning style"): + Capabilities.from_dict( + { + "reasoning": {"style": "telepathy"}, + "media": {"audio": "gemini_part", "image": "gemini_part"}, + "response": "gemini_text", + } + ) diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..27903b5 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,121 @@ +"""Target loading + the capability merge chain. A target YAML declares only what +differs from its provider's defaults; per-benchmark overrides and CLI overrides +layer on top. build_adapter maps the provider to the right adapter class. +""" + +import textwrap + +from bench_core.capabilities import AudioShape, ReasoningStyle +from bench_core.config import build_adapter, load_target, resolve_capabilities +from bench_core.providers.gemini import GeminiAdapter +from bench_core.providers.openai_compat import OpenAICompatAdapter + + +def _write(tmp_path, text): + p = tmp_path / "target.yaml" + p.write_text(textwrap.dedent(text)) + return p + + +def test_load_target_parses_core_fields(tmp_path): + t = load_target( + _write( + tmp_path, + """ + name: inkling + provider: fireworks + model_id: accounts/fireworks/models/inkling + """, + ) + ) + assert t.name == "inkling" + assert t.provider == "fireworks" + assert t.model_id == "accounts/fireworks/models/inkling" + + +def test_target_inherits_provider_defaults(tmp_path): + # A fireworks target declares no media; it should inherit audio_url + the + # "none is a floor" reasoning semantics from the provider defaults. + t = load_target( + _write( + tmp_path, + """ + name: inkling + provider: fireworks + model_id: x + """, + ) + ) + caps = resolve_capabilities(t) + assert caps.media.audio is AudioShape.AUDIO_URL + assert caps.reasoning.style is ReasoningStyle.EFFORT + assert caps.reasoning.true_off is False # Fireworks "none" still thinks + + +def test_target_overrides_win_over_defaults(tmp_path): + # gemini-2.5-flash uses the budget knob, overriding the thinking_level default. + t = load_target( + _write( + tmp_path, + """ + name: gemini-2.5-flash + provider: gemini + model_id: gemini-2.5-flash + capabilities: + reasoning: + style: thinking_budget + off_value: 0 + on_value: -1 + true_off: true + """, + ) + ) + caps = resolve_capabilities(t) + assert caps.reasoning.style is ReasoningStyle.THINKING_BUDGET + assert caps.reasoning.off_value == 0 + + +def test_per_benchmark_override_applies(tmp_path): + t = load_target( + _write( + tmp_path, + """ + name: gemini-3.7-flash + provider: gemini + model_id: gemini-3.7-flash + capabilities: + reasoning: { style: thinking_level, off_value: low, on_value: high, true_off: false } + overrides: + asr: + reasoning: { off_value: high } + """, + ) + ) + assert resolve_capabilities(t, benchmark="gpqa").reasoning.off_value == "low" + assert resolve_capabilities(t, benchmark="asr").reasoning.off_value == "high" + + +def test_cli_override_wins_over_everything(tmp_path): + t = load_target( + _write( + tmp_path, + """ + name: inkling + provider: fireworks + model_id: x + """, + ) + ) + caps = resolve_capabilities(t, cli_overrides={"reasoning": {"on_value": "max"}}) + assert caps.reasoning.on_value == "max" + + +def test_build_adapter_maps_provider_to_class(tmp_path): + fw = load_target(_write(tmp_path, "name: a\nprovider: fireworks\nmodel_id: x\n")) + a = build_adapter(fw) + assert isinstance(a, OpenAICompatAdapter) + assert "fireworks_api_key" in [k.lower() for k in a.key_spec] + assert a.base_url and "fireworks" in a.base_url + + gtarget = load_target(_write(tmp_path, "name: b\nprovider: gemini\nmodel_id: y\n")) + assert isinstance(build_adapter(gtarget), GeminiAdapter) diff --git a/tests/test_decode.py b/tests/test_decode.py new file mode 100644 index 0000000..15d2512 --- /dev/null +++ b/tests/test_decode.py @@ -0,0 +1,86 @@ +"""Response normalization — unify the three response shapes and the +reasoning-token fallbacks the audit found (Fireworks reports reasoning tokens at +the top level of usage; Gemini as thoughts_token_count). Duck-typed fakes stand +in for the SDK objects so this stays a pure unit test. +""" + +from types import SimpleNamespace as NS + +from bench_core.capabilities import ResponseShape +from bench_core.decode import decode_response + + +def _openai_raw( + content, reasoning_tokens=None, top_level_reasoning=None, finish="stop" +): + details = NS(reasoning_tokens=reasoning_tokens) + usage = NS( + completion_tokens_details=details, + reasoning_tokens=top_level_reasoning, + prompt_tokens=11, + completion_tokens=22, + ) + msg = NS(content=content) + choice = NS(message=msg, finish_reason=finish) + return NS(choices=[choice], usage=usage, id="resp_1") + + +def test_openai_extracts_content_and_finish_reason(): + r = decode_response(_openai_raw("Answer: B"), ResponseShape.OPENAI_CHAT) + assert r.text == "Answer: B" + assert r.finish_reason == "stop" + assert r.raw_id == "resp_1" + + +def test_openai_none_content_becomes_empty_string(): + r = decode_response(_openai_raw(None), ResponseShape.OPENAI_CHAT) + assert r.text == "" + + +def test_openai_reasoning_tokens_prefers_details(): + r = decode_response( + _openai_raw("x", reasoning_tokens=729), ResponseShape.OPENAI_CHAT + ) + assert r.reasoning_tokens == 729 + + +def test_openai_reasoning_tokens_falls_back_to_top_level_for_fireworks(): + r = decode_response( + _openai_raw("x", reasoning_tokens=None, top_level_reasoning=512), + ResponseShape.OPENAI_CHAT, + ) + assert r.reasoning_tokens == 512 + + +def test_anthropic_joins_text_blocks_and_skips_thinking(): + raw = NS( + content=[ + NS(type="thinking", thinking="hmm"), + NS(type="text", text="Hello "), + NS(type="text", text="world"), + ], + usage=NS(input_tokens=5, output_tokens=7), + id="msg_1", + ) + r = decode_response(raw, ResponseShape.ANTHROPIC_BLOCKS) + assert r.text == "Hello world" + assert r.input_tokens == 5 + assert r.output_tokens == 7 + + +def test_gemini_reads_text_and_thoughts_token_count(): + raw = NS( + text="42", + usage_metadata=NS(thoughts_token_count=88, prompt_token_count=3), + response_id="gem_1", + ) + r = decode_response(raw, ResponseShape.GEMINI_TEXT) + assert r.text == "42" + assert r.reasoning_tokens == 88 + assert r.raw_id == "gem_1" + + +def test_gemini_none_text_becomes_empty(): + raw = NS(text=None, usage_metadata=NS(thoughts_token_count=None), response_id="g") + r = decode_response(raw, ResponseShape.GEMINI_TEXT) + assert r.text == "" diff --git a/tests/test_encode_anthropic.py b/tests/test_encode_anthropic.py new file mode 100644 index 0000000..f166b6b --- /dev/null +++ b/tests/test_encode_anthropic.py @@ -0,0 +1,71 @@ +"""Anthropic encode: system-message separation, the disabled/enabled thinking +block, temperature omission under thinking, and the max_tokens > budget_tokens +coupling (all declarative via the cap). +""" + +from bench_core.capabilities import Capabilities +from bench_core.providers.anthropic import AnthropicAdapter +from bench_core.request import ImagePart, Message, ReasoningSpec, Request, TextPart + +CAPS = Capabilities.from_dict( + { + "reasoning": { + "style": "disabled_block", + "on_value": 10000, + "true_off": True, + "temperature_when_on": False, + "extra": {"off_max_tokens": 1024, "on_max_tokens": 16000}, + }, + "media": {"image": "anthropic_source"}, + "response": "anthropic_blocks", + } +) +ADAPTER = AnthropicAdapter( + name="anthropic", + base_url=None, + key_spec=["ANTHROPIC_API_KEY"], + capability_defaults={}, +) + + +def test_off_disables_thinking_and_keeps_temperature(): + req = Request( + [Message("user", [TextPart("hi")])], ReasoningSpec("off"), temperature=0.0 + ) + kw = ADAPTER.encode(req, CAPS, "claude-sonnet-4-6") + assert kw["thinking"] == {"type": "disabled"} + assert kw["max_tokens"] == 1024 + assert kw["temperature"] == 0.0 + + +def test_high_enables_budget_bumps_max_tokens_and_omits_temperature(): + req = Request( + [Message("user", [TextPart("hi")])], ReasoningSpec("high"), temperature=0.0 + ) + kw = ADAPTER.encode(req, CAPS, "claude-sonnet-4-6") + assert kw["thinking"] == {"type": "enabled", "budget_tokens": 10000} + assert kw["max_tokens"] > 10000 # must exceed budget + assert "temperature" not in kw + + +def test_system_message_is_separated(): + req = Request( + [ + Message("system", [TextPart("You are terse.")]), + Message("user", [TextPart("hi")]), + ], + ReasoningSpec("off"), + ) + kw = ADAPTER.encode(req, CAPS, "claude-sonnet-4-6") + assert kw["system"] == "You are terse." + assert [m["role"] for m in kw["messages"]] == ["user"] + + +def test_image_uses_anthropic_source_block(): + req = Request( + [Message("user", [TextPart("read"), ImagePart(b"\xff\xd8j", "image/jpeg")])], + ReasoningSpec("off"), + ) + content = ADAPTER.encode(req, CAPS, "claude-sonnet-4-6")["messages"][0]["content"] + assert content[1]["type"] == "image" + assert content[1]["source"]["type"] == "base64" diff --git a/tests/test_encode_gemini.py b/tests/test_encode_gemini.py new file mode 100644 index 0000000..173264a --- /dev/null +++ b/tests/test_encode_gemini.py @@ -0,0 +1,59 @@ +"""Gemini encode: native genai config object. Asserts on the config's thinking +field (level vs budget) and that media parts become genai Parts. Uses the real +google-genai types (offline — no network).""" + +from bench_core.capabilities import Capabilities +from bench_core.providers.gemini import GeminiAdapter +from bench_core.request import ImagePart, Message, ReasoningSpec, Request, TextPart + +LEVEL_CAPS = Capabilities.from_dict( + { + "reasoning": { + "style": "thinking_level", + "off_value": "low", + "on_value": "high", + "true_off": False, + }, + "media": {"image": "gemini_part"}, + "response": "gemini_text", + } +) +BUDGET_CAPS = Capabilities.from_dict( + { + "reasoning": {"style": "thinking_budget", "off_value": 0, "on_value": -1, "true_off": True}, + "media": {"image": "gemini_part"}, + "response": "gemini_text", + } +) +ADAPTER = GeminiAdapter( + name="gemini", base_url=None, key_spec=["GEMINI_API_KEY"], capability_defaults={} +) + + +def test_thinking_level_floor_goes_into_config(): + req = Request( + [Message("user", [TextPart("hi")])], ReasoningSpec("off"), temperature=0.0 + ) + enc = ADAPTER.encode(req, LEVEL_CAPS, "gemini-3.7-flash") + assert enc["model"] == "gemini-3.7-flash" + # the SDK coerces "low" -> ThinkingLevel.LOW enum; compare on value, case-insensitive + assert str(enc["config"].thinking_config.thinking_level.value).lower() == "low" + assert enc["config"].temperature == 0.0 + + +def test_thinking_budget_zero_goes_into_config(): + req = Request([Message("user", [TextPart("hi")])], ReasoningSpec("off")) + enc = ADAPTER.encode(req, BUDGET_CAPS, "gemini-2.5-flash") + assert enc["config"].thinking_config.thinking_budget == 0 + + +def test_image_part_becomes_genai_part(): + req = Request( + [Message("user", [TextPart("read"), ImagePart(b"\x89PNGx", "image/png")])], + ReasoningSpec("off"), + ) + contents = ADAPTER.encode(req, LEVEL_CAPS, "gemini-3.7-flash")["contents"] + # text passes through as a string; the image becomes a genai Part with inline bytes + assert "read" in contents + part = contents[-1] + assert getattr(part, "inline_data", None) is not None diff --git a/tests/test_encode_openai_compat.py b/tests/test_encode_openai_compat.py new file mode 100644 index 0000000..a757f37 --- /dev/null +++ b/tests/test_encode_openai_compat.py @@ -0,0 +1,122 @@ +"""OpenAI-compatible encode — the assembly point where reasoning placement, +temperature gating, media blocks and the max-tokens param name all come +together for openai / fireworks / openrouter / interfaze. +""" + +from bench_core.capabilities import Capabilities +from bench_core.providers.openai_compat import OpenAICompatAdapter +from bench_core.request import ImagePart, Message, ReasoningSpec, Request, TextPart + +OPENAI_CAPS = Capabilities.from_dict( + { + "reasoning": { + "style": "effort", + "off_value": "none", + "on_value": "high", + "true_off": True, + "temperature_when_on": False, + }, + "media": {"image": "image_url"}, + "response": "openai_chat", + } +) +FIREWORKS_CAPS = Capabilities.from_dict( + { + "reasoning": { + "style": "effort", + "off_value": "none", + "on_value": "high", + "true_off": False, + "temperature_when_on": True, + }, + "media": {"image": "image_url"}, + "response": "openai_chat", + "max_tokens_param": "max_completion_tokens", + } +) +OPENROUTER_CAPS = Capabilities.from_dict( + { + "reasoning": { + "style": "reasoning_body", + "off_value": False, + "on_value": True, + "true_off": True, + "extra": {"toggle": "enabled"}, + }, + "media": {"image": "image_url"}, + "response": "openai_chat", + } +) + +ADAPTER = OpenAICompatAdapter( + name="test", base_url="http://x", key_spec=["K"], capability_defaults={} +) + + +def _text_req(mode, temperature=0.0): + return Request( + messages=[Message("user", [TextPart("What is 2+2?")])], + reasoning=ReasoningSpec(mode=mode), + temperature=temperature, + ) + + +def test_text_only_content_is_a_plain_string(): + kw = ADAPTER.encode(_text_req("off"), OPENAI_CAPS, "gpt-5.5") + assert kw["model"] == "gpt-5.5" + assert kw["messages"] == [{"role": "user", "content": "What is 2+2?"}] + + +def test_openai_off_sends_effort_and_temperature(): + kw = ADAPTER.encode(_text_req("off"), OPENAI_CAPS, "gpt-5.5") + assert kw["reasoning_effort"] == "none" + assert kw["temperature"] == 0.0 + + +def test_openai_high_omits_temperature(): + kw = ADAPTER.encode(_text_req("high"), OPENAI_CAPS, "gpt-5.5") + assert kw["reasoning_effort"] == "high" + assert "temperature" not in kw + + +def test_fireworks_high_keeps_temperature(): + kw = ADAPTER.encode(_text_req("high"), FIREWORKS_CAPS, "inkling") + assert kw["reasoning_effort"] == "high" + assert kw["temperature"] == 0.0 + + +def test_max_tokens_uses_capability_param_name(): + req = _text_req("off") + req.max_tokens = 4096 + assert ( + ADAPTER.encode(req, FIREWORKS_CAPS, "inkling")["max_completion_tokens"] == 4096 + ) + assert "max_tokens" not in ADAPTER.encode(req, FIREWORKS_CAPS, "inkling") + + +def test_openrouter_reasoning_goes_to_extra_body(): + kw = ADAPTER.encode(_text_req("off"), OPENROUTER_CAPS, "x-ai/grok-4.3") + assert "reasoning_effort" not in kw + assert kw["extra_body"] == {"reasoning": {"enabled": False}} + + +def test_image_message_becomes_block_list(): + req = Request( + messages=[ + Message( + "user", + [TextPart("Read this."), ImagePart(b"\xff\xd8jpg", "image/jpeg")], + ) + ], + reasoning=ReasoningSpec(mode="off"), + ) + content = ADAPTER.encode(req, OPENAI_CAPS, "gpt-5.5")["messages"][0]["content"] + assert isinstance(content, list) + assert content[0] == {"type": "text", "text": "Read this."} + assert content[1]["type"] == "image_url" + assert content[1]["image_url"]["url"].startswith("data:image/jpeg;base64,") + + +def test_none_temperature_is_omitted(): + kw = ADAPTER.encode(_text_req("off", temperature=None), OPENAI_CAPS, "gpt-5.5") + assert "temperature" not in kw diff --git a/tests/test_errors.py b/tests/test_errors.py new file mode 100644 index 0000000..025bc2d --- /dev/null +++ b/tests/test_errors.py @@ -0,0 +1,75 @@ +"""Error classification — turns a provider exception into the signal the +fallback ladder acts on. Every PARAM_REJECTED case here is a real 4xx the audit +saw a model raise (Gemini "MINIMAL not supported", grok "Reasoning is +mandatory", GPT-5 temperature-under-reasoning). +""" + +from bench_core.errors import ErrorKind, classify + + +class FakeAPIError(Exception): + """Mimics an SDK error carrying a status code + message.""" + + def __init__(self, status_code, message): + super().__init__(message) + self.status_code = status_code + + +def test_429_is_rate_limited(): + assert ( + classify(FakeAPIError(429, "Too Many Requests")).kind is ErrorKind.RATE_LIMITED + ) + + +def test_503_is_transient(): + assert ( + classify(FakeAPIError(503, "Service Unavailable")).kind is ErrorKind.TRANSIENT + ) + + +def test_timeout_is_transient(): + assert classify(TimeoutError("request timed out")).kind is ErrorKind.TRANSIENT + + +def test_401_is_fatal(): + assert classify(FakeAPIError(401, "invalid api key")).kind is ErrorKind.FATAL + + +def test_404_model_not_found_is_fatal(): + assert classify(FakeAPIError(404, "model does not exist")).kind is ErrorKind.FATAL + + +def test_gemini_minimal_not_supported_raises_thinking_floor(): + # Raised with no status_code attr — parsed from the message, as google-genai does. + c = classify( + Exception( + "400 INVALID_ARGUMENT. Thinking level MINIMAL is not supported for this model." + ) + ) + assert c.kind is ErrorKind.PARAM_REJECTED + assert c.action == "raise_thinking_floor" + + +def test_reasoning_mandatory_forces_floor(): + c = classify(FakeAPIError(400, "Reasoning is mandatory and cannot be disabled")) + assert c.kind is ErrorKind.PARAM_REJECTED + assert c.action == "force_reasoning_floor" + + +def test_temperature_rejected_drops_temperature(): + c = classify( + FakeAPIError(400, "temperature is not supported with reasoning enabled") + ) + assert c.kind is ErrorKind.PARAM_REJECTED + assert c.action == "drop_temperature" + + +def test_unknown_parameter_drops_that_param(): + c = classify(FakeAPIError(400, "Unknown parameter: 'top_k'")) + assert c.kind is ErrorKind.PARAM_REJECTED + assert c.action == "drop_param" + assert c.param == "top_k" + + +def test_unrecognized_400_is_fatal(): + assert classify(FakeAPIError(400, "malformed request body")).kind is ErrorKind.FATAL diff --git a/tests/test_execute.py b/tests/test_execute.py new file mode 100644 index 0000000..65107c4 --- /dev/null +++ b/tests/test_execute.py @@ -0,0 +1,162 @@ +"""The fallback ladder. On a PARAM_REJECTED 4xx it mutates the request/caps and +retries in-place (self-heal), recording a capability_hint to promote to YAML. +Transient/rate-limit/empty/fatal are surfaced as BenchError for the runner to +retry-or-fail. A scripted fake adapter drives the error sequence — no network. +""" + +import pytest + +from bench_core.capabilities import Capabilities +from bench_core.errors import ErrorKind +from bench_core.execute import BenchError, execute +from bench_core.request import Message, ReasoningSpec, Request, TextPart +from bench_core.response import Response + + +class FakeAPIError(Exception): + def __init__(self, status_code, message): + super().__init__(message) + self.status_code = status_code + + +class ScriptedAdapter: + """Records the encoded state on each call so tests can assert what the + ladder adjusted; raises a scripted error sequence, then succeeds.""" + + def __init__(self, errors, final_text="ok"): + self.errors = list(errors) + self.history = [] + self.final_text = final_text + + def encode(self, req, caps, model_id): + return { + "mode": req.reasoning.mode, + "off": caps.reasoning.off_value, + "toggle": caps.reasoning.extra.get("toggle"), + "true_off": caps.reasoning.true_off, + "temperature": req.temperature, + } + + def call(self, client, encoded): + self.history.append(encoded) + if self.errors: + raise self.errors.pop(0) + return {"ok": True} + + def decode(self, raw, caps): + return Response(text=self.final_text) + + def classify_error(self, exc): + from bench_core.errors import classify + + return classify(exc) + + +def _req(mode="off", temperature=0.0): + return Request( + [Message("user", [TextPart("q")])], ReasoningSpec(mode), temperature=temperature + ) + + +def _caps(**reasoning): + base = { + "style": "thinking_level", + "off_value": "minimal", + "on_value": "high", + "true_off": False, + } + base.update(reasoning) + return Capabilities.from_dict( + {"reasoning": base, "media": {}, "response": "gemini_text"} + ) + + +def test_success_first_try_returns_response_no_hints(): + adapter = ScriptedAdapter(errors=[]) + resp, hints = execute(adapter, None, _req(), _caps(), "m") + assert resp.text == "ok" + assert hints == [] + assert len(adapter.history) == 1 + + +def test_minimal_not_supported_raises_thinking_floor_then_succeeds(): + adapter = ScriptedAdapter( + errors=[ + Exception( + "400 INVALID_ARGUMENT. Thinking level MINIMAL is not supported for this model." + ) + ] + ) + resp, hints = execute(adapter, None, _req(), _caps(off_value="minimal"), "m") + assert resp.text == "ok" + assert ( + adapter.history[0]["off"] == "minimal" + ) # first attempt used the declared floor + assert adapter.history[1]["off"] == "low" # ladder raised it + assert any("floor" in h.lower() for h in hints) + + +def test_temperature_rejected_drops_temperature_then_succeeds(): + adapter = ScriptedAdapter( + errors=[FakeAPIError(400, "temperature is not supported with reasoning")] + ) + _resp, hints = execute(adapter, None, _req(temperature=0.0), _caps(), "m") + assert adapter.history[0]["temperature"] == 0.0 + assert adapter.history[1]["temperature"] is None + assert any("temperature" in h.lower() for h in hints) + + +def test_reasoning_mandatory_forces_floor_on_reasoning_body(): + caps = Capabilities.from_dict( + { + "reasoning": { + "style": "reasoning_body", + "off_value": False, + "on_value": True, + "true_off": True, + "extra": {"toggle": "enabled"}, + }, + "media": {}, + "response": "openai_chat", + } + ) + adapter = ScriptedAdapter( + errors=[FakeAPIError(400, "Reasoning is mandatory and cannot be disabled")] + ) + _resp, hints = execute(adapter, None, _req(), caps, "m") + assert adapter.history[0]["toggle"] == "enabled" + assert adapter.history[1]["toggle"] == "effort" # switched away from enabled=false + assert any("reasoning" in h.lower() for h in hints) + + +def test_fatal_error_propagates_as_bencherror(): + adapter = ScriptedAdapter(errors=[FakeAPIError(401, "bad key")]) + with pytest.raises(BenchError) as ei: + execute(adapter, None, _req(), _caps(), "m") + assert ei.value.classification.kind is ErrorKind.FATAL + + +def test_empty_content_surfaces_as_retryable_bencherror(): + adapter = ScriptedAdapter(errors=[], final_text=" ") + with pytest.raises(BenchError) as ei: + execute(adapter, None, _req(), _caps(), "m") + assert ei.value.classification.kind is ErrorKind.EMPTY_CONTENT + + +def test_rate_limit_surfaces_as_retryable_bencherror(): + adapter = ScriptedAdapter(errors=[FakeAPIError(429, "slow down")]) + with pytest.raises(BenchError) as ei: + execute(adapter, None, _req(), _caps(), "m") + assert ei.value.classification.kind is ErrorKind.RATE_LIMITED + + +def test_param_retries_are_bounded(): + # A model that rejects MINIMAL forever must not loop indefinitely. + adapter = ScriptedAdapter( + errors=[Exception("Thinking level X is not supported")] * 10 + ) + with pytest.raises(BenchError): + execute( + adapter, None, _req(), _caps(off_value="minimal"), "m", max_param_retries=3 + ) + assert len(adapter.history) <= 4 diff --git a/tests/test_media.py b/tests/test_media.py new file mode 100644 index 0000000..9429ac7 --- /dev/null +++ b/tests/test_media.py @@ -0,0 +1,118 @@ +"""Media block shapes — the audit's most dangerous fragmentation. Four audio +shapes and several image shapes look almost interchangeable but are host- +specific and fail *silently* when wrong (an audio_url data-URI sent to +OpenRouter is accepted, the audio dropped, and the model hallucinates a +transcript at WER~1.0). These tests pin each shape exactly. +""" + +import base64 +import io + +from bench_core.capabilities import AudioShape, ImageShape +from bench_core.media import GeminiPart, audio_block, image_block +from bench_core.request import AudioPart, ImagePart + +WAV = b"RIFF....WAVEfake-audio-bytes" +JPEG = b"\xff\xd8\xff\xe0jpeg-bytes" +PNG = b"\x89PNG\r\n\x1a\npng-bytes" + + +def _b64(data: bytes) -> str: + return base64.b64encode(data).decode("ascii") + + +# --- audio --------------------------------------------------------------- + + +def test_audio_file_block_is_interfaze_data_uri(): + block = audio_block(AudioPart(WAV, "audio/wav"), AudioShape.FILE_BLOCK) + assert block["type"] == "file" + assert block["file"]["file_data"] == f"data:audio/wav;base64,{_b64(WAV)}" + + +def test_audio_url_block_is_fireworks_data_uri(): + block = audio_block(AudioPart(WAV, "audio/wav"), AudioShape.AUDIO_URL) + assert block == { + "type": "audio_url", + "audio_url": {"url": f"data:audio/wav;base64,{_b64(WAV)}"}, + } + + +def test_input_audio_block_is_openrouter_raw_b64_with_format(): + # This is the anti-silent-drop shape: raw base64 (NOT a data-URI) + format. + block = audio_block(AudioPart(WAV, "audio/wav"), AudioShape.INPUT_AUDIO) + assert block == { + "type": "input_audio", + "input_audio": {"data": _b64(WAV), "format": "wav"}, + } + assert "data:" not in block["input_audio"]["data"] + + +def test_audio_gemini_part_carries_bytes_and_mime(): + block = audio_block(AudioPart(WAV, "audio/wav"), AudioShape.GEMINI_PART) + assert block == GeminiPart(data=WAV, mime="audio/wav") + + +# --- image --------------------------------------------------------------- + + +def test_image_url_block_jpeg_data_uri(): + block = image_block(ImagePart(JPEG, "image/jpeg"), ImageShape.IMAGE_URL) + assert block == { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{_b64(JPEG)}"}, + } + + +def test_image_url_block_preserves_png_mime_for_olmocr(): + block = image_block(ImagePart(PNG, "image/png"), ImageShape.IMAGE_URL) + assert block["image_url"]["url"].startswith("data:image/png;base64,") + + +def test_image_anthropic_source_block(): + block = image_block(ImagePart(JPEG, "image/jpeg"), ImageShape.ANTHROPIC_SOURCE) + assert block == { + "type": "image", + "source": {"type": "base64", "media_type": "image/jpeg", "data": _b64(JPEG)}, + } + + +def test_image_gemini_part_marker(): + block = image_block(ImagePart(JPEG, "image/jpeg"), ImageShape.GEMINI_PART) + assert block == GeminiPart(data=JPEG, mime="image/jpeg") + + +# --- encode_image (central RGB-convert + resize) ------------------------- + + +def test_encode_image_converts_rgba_to_jpeg_without_crashing(): + # PIL raises OSError saving RGBA as JPEG; the convert must happen here so a + # transparent PNG sample doesn't crash the run (it did in 8/9 OCR variants). + from PIL import Image + + from bench_core.media import encode_image + + rgba = Image.new("RGBA", (10, 10), (255, 0, 0, 128)) + part = encode_image(rgba, mime="image/jpeg") + assert part.mime == "image/jpeg" + assert Image.open(io.BytesIO(part.data)).mode == "RGB" + + +def test_encode_image_downscales_to_max_side(): + from PIL import Image + + from bench_core.media import encode_image + + big = Image.new("RGB", (3000, 1500)) + part = encode_image(big, mime="image/jpeg", max_side=1536) + assert max(Image.open(io.BytesIO(part.data)).size) == 1536 + + +def test_encode_image_png_keeps_alpha(): + from PIL import Image + + from bench_core.media import encode_image + + rgba = Image.new("RGBA", (10, 10), (0, 255, 0, 64)) + part = encode_image(rgba, mime="image/png") + assert Image.open(io.BytesIO(part.data)).mode == "RGBA" diff --git a/tests/test_reasoning.py b/tests/test_reasoning.py new file mode 100644 index 0000000..6ec9ff5 --- /dev/null +++ b/tests/test_reasoning.py @@ -0,0 +1,168 @@ +"""The reasoning translation — the single place the audit's "6 different ways" +of encoding a thinking floor collapse into. Every case here is a real quirk +that currently lives as a scattered special-case: + +- OpenAI: reasoning_effort="none", and temperature only when reasoning is off +- Fireworks: reasoning_effort="none" is a FLOOR (still thinks), temp always sent +- Gemini 3.7-flash: thinking_level floor "low" (rejects "minimal") +- Gemini 3.x-flash <3.7: thinking_level floor "minimal" +- Gemini 2.5-flash: thinking_budget=0 (true off); 2.5-pro: budget=128 (floor) +- Anthropic: thinking={"type":"disabled"} off / enabled+budget on, temp omitted on +- OpenRouter default: reasoning.enabled bool; grok/google: reasoning.effort (can't disable) +- Moonshot: reasoning.enabled=false + provider pin +""" + +from bench_core.capabilities import ReasoningCap, ReasoningStyle +from bench_core.reasoning import build_reasoning + + +def test_none_style_injects_nothing(): + cap = ReasoningCap(style=ReasoningStyle.NONE) + inj = build_reasoning("high", cap) + assert inj.kwargs == {} + assert inj.extra_body == {} + assert inj.thinking_level is None + assert inj.thinking_budget is None + assert inj.anthropic_thinking is None + assert inj.allow_temperature is True + + +def test_openai_effort_off_sends_none_and_allows_temperature(): + # OpenAI "none" is a true off, so temperature is allowed on the off run. + cap = ReasoningCap( + style=ReasoningStyle.EFFORT, + off_value="none", + on_value="high", + true_off=True, + temperature_when_on=False, + ) + inj = build_reasoning("off", cap) + assert inj.kwargs == {"reasoning_effort": "none"} + assert inj.allow_temperature is True + + +def test_openai_effort_high_omits_temperature(): + # GPT-5.x rejects temperature!=default once reasoning engages. + cap = ReasoningCap( + style=ReasoningStyle.EFFORT, + off_value="none", + on_value="high", + true_off=True, + temperature_when_on=False, + ) + inj = build_reasoning("high", cap) + assert inj.kwargs == {"reasoning_effort": "high"} + assert inj.allow_temperature is False + + +def test_fireworks_effort_floor_still_allows_temperature_on_high(): + # Fireworks/Inkling sends temperature regardless of reasoning. + cap = ReasoningCap( + style=ReasoningStyle.EFFORT, + off_value="none", + on_value="high", + true_off=False, + temperature_when_on=True, + ) + inj = build_reasoning("high", cap) + assert inj.kwargs == {"reasoning_effort": "high"} + assert inj.allow_temperature is True + + +def test_effort_intermediate_mode_passes_through(): + cap = ReasoningCap(style=ReasoningStyle.EFFORT, off_value="none", on_value="high") + assert build_reasoning("low", cap).kwargs == {"reasoning_effort": "low"} + + +def test_gemini_37_flash_thinking_level_floor_is_low(): + cap = ReasoningCap( + style=ReasoningStyle.THINKING_LEVEL, off_value="low", on_value="high", true_off=False + ) + inj = build_reasoning("off", cap) + assert inj.thinking_level == "low" + assert inj.thinking_budget is None + + +def test_gemini_old_flash_thinking_level_floor_is_minimal(): + cap = ReasoningCap( + style=ReasoningStyle.THINKING_LEVEL, off_value="minimal", on_value="high", true_off=False + ) + assert build_reasoning("off", cap).thinking_level == "minimal" + + +def test_gemini_25_flash_budget_zero_is_true_off(): + cap = ReasoningCap( + style=ReasoningStyle.THINKING_BUDGET, off_value=0, on_value=-1, true_off=True + ) + inj = build_reasoning("off", cap) + assert inj.thinking_budget == 0 + assert inj.thinking_level is None + + +def test_gemini_25_pro_budget_floor_is_128(): + cap = ReasoningCap( + style=ReasoningStyle.THINKING_BUDGET, off_value=128, on_value=-1, true_off=False + ) + assert build_reasoning("off", cap).thinking_budget == 128 + assert build_reasoning("high", cap).thinking_budget == -1 + + +def test_anthropic_disabled_block_off_allows_temperature(): + cap = ReasoningCap( + style=ReasoningStyle.DISABLED_BLOCK, + on_value=10000, + true_off=True, + temperature_when_on=False, + ) + inj = build_reasoning("off", cap) + assert inj.anthropic_thinking == {"type": "disabled"} + assert inj.allow_temperature is True + + +def test_anthropic_disabled_block_on_enables_budget_and_omits_temperature(): + cap = ReasoningCap( + style=ReasoningStyle.DISABLED_BLOCK, + on_value=10000, + true_off=True, + temperature_when_on=False, + ) + inj = build_reasoning("high", cap) + assert inj.anthropic_thinking == {"type": "enabled", "budget_tokens": 10000} + assert inj.allow_temperature is False + + +def test_openrouter_enabled_toggle_off_disables(): + cap = ReasoningCap( + style=ReasoningStyle.REASONING_BODY, + off_value=False, + on_value=True, + true_off=True, + extra={"toggle": "enabled"}, + ) + inj = build_reasoning("off", cap) + assert inj.extra_body == {"reasoning": {"enabled": False}} + + +def test_openrouter_effort_toggle_cannot_disable_uses_minimal_floor(): + # grok/google via OpenRouter reject enabled=false; floor is effort=minimal. + cap = ReasoningCap( + style=ReasoningStyle.REASONING_BODY, + off_value="minimal", + on_value="high", + true_off=False, + extra={"toggle": "effort"}, + ) + inj = build_reasoning("off", cap) + assert inj.extra_body == {"reasoning": {"effort": "minimal"}} + + +def test_openrouter_provider_pin_is_emitted(): + cap = ReasoningCap( + style=ReasoningStyle.REASONING_BODY, + off_value=False, + on_value=True, + extra={"toggle": "enabled", "provider_only": ["moonshotai"]}, + ) + inj = build_reasoning("off", cap) + assert inj.extra_body["reasoning"] == {"enabled": False} + assert inj.extra_body["provider"] == {"only": ["moonshotai"]} diff --git a/tests/test_results.py b/tests/test_results.py new file mode 100644 index 0000000..638c485 --- /dev/null +++ b/tests/test_results.py @@ -0,0 +1,57 @@ +"""The single result contract: one slug, one layout +(results///{responses.jsonl,metrics.json,run.json}), and ONE +content-based discovery so reporting and re-scoring can't disagree (the audit +found report_scores dropping 5 of 7 RefCOCO runners by prefix mismatch). +""" + +from bench_core.results import RunStore, discover, model_slug + + +def test_model_slug_is_canonical(): + assert model_slug("accounts/fireworks/models/inkling-small") == "inkling-small" + assert model_slug("google/gemini-3.7-flash") == "gemini-3.7-flash" + assert model_slug("gpt-5.5") == "gpt-5.5" + assert model_slug("x-ai/grok-4.3") == "grok-4.3" + + +def test_append_and_load_responses_round_trip(tmp_path): + store = RunStore("gpqa", "inkling", root=tmp_path) + store.append_response({"id": "a", "response": "A"}) + store.append_response({"id": "b", "response": "B"}) + assert store.load_responses() == [ + {"id": "a", "response": "A"}, + {"id": "b", "response": "B"}, + ] + + +def test_completed_ids_uses_done_predicate(tmp_path): + store = RunStore("gpqa", "inkling", root=tmp_path) + store.append_response({"id": "a", "response": "A"}) + store.append_response({"id": "b", "response": None}) # not done + done = store.completed_ids("id", lambda r: r.get("response") is not None) + assert done == {"a"} + + +def test_write_metrics_and_run(tmp_path): + store = RunStore("gpqa", "inkling", root=tmp_path) + store.write_metrics({"benchmark": "gpqa", "score": 0.87}) + store.write_run({"provider": "fireworks", "model_id": "x"}) + assert store.metrics_path.exists() + assert store.run_path.exists() + + +def test_discover_finds_all_metrics_content_based(tmp_path): + RunStore("gpqa", "inkling", root=tmp_path).write_metrics( + {"benchmark": "gpqa", "target": "inkling", "score": 0.87} + ) + RunStore("refcoco", "gemini-3.7-flash", root=tmp_path).write_metrics( + {"benchmark": "refcoco", "target": "gemini-3.7-flash", "score": 0.3} + ) + found = discover(root=tmp_path) + keys = {(m["benchmark"], m["target"]) for m in found} + assert keys == {("gpqa", "inkling"), ("refcoco", "gemini-3.7-flash")} + + +def test_paths_live_under_benchmark_and_slugged_target(tmp_path): + store = RunStore("mmmu_pro", "accounts/fireworks/models/inkling", root=tmp_path) + assert store.dir == tmp_path / "mmmu_pro" / "inkling" diff --git a/tests/test_runner.py b/tests/test_runner.py new file mode 100644 index 0000000..6ca6e42 --- /dev/null +++ b/tests/test_runner.py @@ -0,0 +1,143 @@ +"""The execution harness: resume, retry-with-backoff on retryable BenchErrors, +fail-fast on fatal, incremental checkpointing, bounded concurrency. Driven by a +scripted fake adapter through the real `execute`, so no network and no sleeps +(backoff_base=0). +""" + +from bench_core.capabilities import Capabilities +from bench_core.request import Message, ReasoningSpec, Request, TextPart +from bench_core.response import Response +from bench_core.results import RunStore +from bench_core.runner import run_benchmark + + +class FakeAPIError(Exception): + def __init__(self, status_code, message): + super().__init__(message) + self.status_code = status_code + + +class MapAdapter: + """Answers per-sample from a scripted map of id -> list of outcomes, where + an outcome is either an Exception (raised) or a str (returned as text).""" + + def __init__(self, script): + self.script = {k: list(v) for k, v in script.items()} + self.calls = {} + + def encode(self, req, caps, model_id): + return {"id": req.messages[0].parts[0].text} + + def call(self, client, encoded): + sid = encoded["id"] + self.calls[sid] = self.calls.get(sid, 0) + 1 + outcome = self.script[sid].pop(0) + if isinstance(outcome, Exception): + raise outcome + return {"text": outcome} + + def decode(self, raw, caps): + return Response(text=raw["text"]) + + def classify_error(self, exc): + from bench_core.errors import classify + + return classify(exc) + + +CAPS = Capabilities.from_dict( + {"reasoning": {"style": "none"}, "media": {}, "response": "openai_chat"} +) + + +def _build_request(sample): + return Request([Message("user", [TextPart(sample["id"])])], ReasoningSpec("off")) + + +def _parse(resp, sample): + return resp.text.upper() + + +async def _run(adapter, samples, store, **kw): + return await run_benchmark( + adapter=adapter, + client=None, + caps=CAPS, + model_id="m", + samples=samples, + build_request=_build_request, + parse=_parse, + store=store, + backoff_base=0.0, + **kw, + ) + + +async def test_runs_all_samples_and_records_predictions(tmp_path): + adapter = MapAdapter({"a": ["alpha"], "b": ["beta"]}) + store = RunStore("t", "m", root=tmp_path) + result = await _run(adapter, [{"id": "a"}, {"id": "b"}], store) + assert result.n_completed == 2 + recs = {r["id"]: r for r in store.load_responses()} + assert recs["a"]["prediction"] == "ALPHA" + assert recs["b"]["response"] == "beta" + + +async def test_resume_skips_completed(tmp_path): + store = RunStore("t", "m", root=tmp_path) + store.append_response({"id": "a", "response": "old", "prediction": "OLD"}) + adapter = MapAdapter( + {"b": ["beta"]} + ) # note: no script for "a" — must not be called + result = await _run(adapter, [{"id": "a"}, {"id": "b"}], store) + assert "a" not in adapter.calls + assert result.n_completed == 1 # only the new one + + +async def test_retryable_error_is_retried_then_succeeds(tmp_path): + adapter = MapAdapter({"a": [FakeAPIError(429, "slow"), "ok"]}) + store = RunStore("t", "m", root=tmp_path) + result = await _run(adapter, [{"id": "a"}], store, max_retries=3) + assert adapter.calls["a"] == 2 + assert result.n_completed == 1 + + +async def test_fatal_error_records_failure_without_retry(tmp_path): + adapter = MapAdapter({"a": [FakeAPIError(401, "bad key")]}) + store = RunStore("t", "m", root=tmp_path) + result = await _run(adapter, [{"id": "a"}], store, max_retries=3) + assert adapter.calls["a"] == 1 # not retried + assert result.n_failed == 1 + assert store.load_responses()[0]["response"] is None + + +async def test_capability_hints_are_aggregated(tmp_path): + adapter = MapAdapter( + {"a": [Exception("Thinking level MINIMAL is not supported"), "ok"]} + ) + store = RunStore("t", "m", root=tmp_path) + # give the reasoning a floor so the ladder has something to raise + caps = Capabilities.from_dict( + { + "reasoning": { + "style": "thinking_level", + "off_value": "minimal", + "on_value": "high", + "true_off": False, + }, + "media": {}, + "response": "openai_chat", + } + ) + result = await run_benchmark( + adapter=adapter, + client=None, + caps=caps, + model_id="m", + samples=[{"id": "a"}], + build_request=_build_request, + parse=_parse, + store=store, + backoff_base=0.0, + ) + assert any("floor" in h.lower() for h in result.hints) diff --git a/tests/test_targets.py b/tests/test_targets.py new file mode 100644 index 0000000..369de0a --- /dev/null +++ b/tests/test_targets.py @@ -0,0 +1,45 @@ +"""Every shipped target YAML must load, resolve to typed capabilities, and build +its adapter — with no network. This is the guard behind the "add a YAML, no +code" promise: a new target file is validated here (and in CI on push) rather +than failing mid-run. +""" + +from pathlib import Path + +import pytest + +from bench_core.capabilities import Capabilities +from bench_core.config import build_adapter, load_target, resolve_capabilities + +TARGET_DIR = Path(__file__).resolve().parent.parent / "bench_core" / "targets" +TARGET_FILES = sorted(TARGET_DIR.glob("*.yaml")) + + +def test_targets_exist(): + assert TARGET_FILES, "no target YAML files found" + + +@pytest.mark.parametrize("path", TARGET_FILES, ids=lambda p: p.stem) +def test_target_loads_resolves_and_builds_adapter(path): + target = load_target(path) + # the filename should match the declared name (keeps the dispatch dropdown honest) + assert target.name == path.stem + caps = resolve_capabilities(target) + assert isinstance(caps, Capabilities) + # per-benchmark override paths must also resolve + resolve_capabilities(target, benchmark="asr") + adapter = build_adapter(target) + assert adapter.name == target.provider + assert adapter.key_spec # some env var to read a key from + + +@pytest.mark.parametrize("path", TARGET_FILES, ids=lambda p: p.stem) +def test_reasoning_off_and_on_values_are_not_yaml_booleans_by_accident(path): + # Guards the off:/on: -> bool trap: if a future edit uses bare off:/on: keys, + # the reasoning dict loses its values and this catches it. + target = load_target(path) + reasoning = target.capabilities.get("reasoning", {}) + assert False not in reasoning and True not in reasoning, ( + f"{path.name} used bare off:/on: keys (parsed as booleans) — " + "use off_value:/on_value:" + ) From e3fdbcb4d0f8d9dcc886cdc8b2a8b098c0d47528 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 07:42:08 +0530 Subject: [PATCH 03/38] refactor(bench_core): consolidate per-model target YAMLs into one targets.yaml One file with a targets: map (key = target name) instead of a file per model. load_all_targets()/load_target(name) read it; tests updated. Same schema, less directory clutter, easier to eyeball. --- bench_core/config.py | 47 ++++-- bench_core/targets.yaml | 77 ++++++++++ bench_core/targets/claude-sonnet-4-6.yaml | 5 - bench_core/targets/gemini-2.5-flash.yaml | 11 -- bench_core/targets/gemini-3.7-flash.yaml | 5 - bench_core/targets/gpt-5.5.yaml | 5 - bench_core/targets/grok-4.3.yaml | 11 -- bench_core/targets/inkling-small.yaml | 10 -- bench_core/targets/inkling.yaml | 9 -- bench_core/targets/interfaze-beta.yaml | 5 - bench_core/targets/kimi-k2.6.yaml | 13 -- tests/test_config.py | 171 +++++++++++++--------- tests/test_targets.py | 44 +++--- 13 files changed, 235 insertions(+), 178 deletions(-) create mode 100644 bench_core/targets.yaml delete mode 100644 bench_core/targets/claude-sonnet-4-6.yaml delete mode 100644 bench_core/targets/gemini-2.5-flash.yaml delete mode 100644 bench_core/targets/gemini-3.7-flash.yaml delete mode 100644 bench_core/targets/gpt-5.5.yaml delete mode 100644 bench_core/targets/grok-4.3.yaml delete mode 100644 bench_core/targets/inkling-small.yaml delete mode 100644 bench_core/targets/inkling.yaml delete mode 100644 bench_core/targets/interfaze-beta.yaml delete mode 100644 bench_core/targets/kimi-k2.6.yaml diff --git a/bench_core/config.py b/bench_core/config.py index 7ed027d..3e65e2a 100644 --- a/bench_core/config.py +++ b/bench_core/config.py @@ -152,21 +152,42 @@ class Target: raw: dict = field(default_factory=dict) -def load_target(path: str | Path) -> Target: +DEFAULT_TARGETS_FILE = Path(__file__).resolve().parent / "targets.yaml" + + +def load_all_targets(path: str | Path = DEFAULT_TARGETS_FILE) -> dict[str, Target]: + """Parse the single targets file into {name: Target}. The map KEY is the + target name — one file, one entry per model, no code to add a model.""" data = yaml.safe_load(Path(path).read_text()) or {} - if data.get("provider") not in PROVIDERS: - raise ValueError( - f"unknown provider {data.get('provider')!r}; known: {sorted(PROVIDERS)}" + raw_targets = data.get("targets") or {} + out: dict[str, Target] = {} + for name, spec in raw_targets.items(): + spec = spec or {} + provider = spec.get("provider") + if provider not in PROVIDERS: + raise ValueError( + f"target {name!r}: unknown provider {provider!r}; " + f"known: {sorted(PROVIDERS)}" + ) + if not spec.get("model_id"): + raise ValueError(f"target {name!r}: missing model_id") + out[name] = Target( + name=name, + provider=provider, + model_id=spec["model_id"], + capabilities=spec.get("capabilities") or {}, + overrides=spec.get("overrides") or {}, + harness_specific=spec.get("harness_specific") or {}, + raw=spec, ) - return Target( - name=data["name"], - provider=data["provider"], - model_id=data["model_id"], - capabilities=data.get("capabilities") or {}, - overrides=data.get("overrides") or {}, - harness_specific=data.get("harness_specific") or {}, - raw=data, - ) + return out + + +def load_target(name: str, path: str | Path = DEFAULT_TARGETS_FILE) -> Target: + targets = load_all_targets(path) + if name not in targets: + raise ValueError(f"unknown target {name!r}; available: {sorted(targets)}") + return targets[name] def resolve_capabilities( diff --git a/bench_core/targets.yaml b/bench_core/targets.yaml new file mode 100644 index 0000000..f322581 --- /dev/null +++ b/bench_core/targets.yaml @@ -0,0 +1,77 @@ +# All benchmark targets in one file. The map KEY is the target name (also the +# result-directory name). Declare only what differs from the provider's defaults +# (see bench_core/config.py PROVIDERS). Adding a model = adding an entry here; no +# code change. +# +# Schema per target: +# provider: fireworks | openrouter | openai | anthropic | gemini | interfaze +# model_id: the id the provider expects +# capabilities: (optional) overrides of the provider defaults +# reasoning: { style, off_value, on_value, true_off, temperature_when_on, extra } +# media: { image, audio } +# response: openai_chat | anthropic_blocks | gemini_text +# overrides: (optional) per-benchmark capability tweaks, e.g. { asr: {...} } +# harness_specific: (optional) opaque config for isolated harnesses, e.g. { bfcl: {...} } +# ci_regression: true (optional) include in the nightly regression matrix +# +# NB: reasoning keys are off_value/on_value, NOT off/on — bare `off:`/`on:` are +# YAML booleans (off: low -> {False: low}). + +targets: + # --- Fireworks: reasoning "none" is a FLOOR (still thinks); audio via audio_url --- + inkling: + provider: fireworks + model_id: accounts/fireworks/models/inkling + + deepseek-v4-flash: + provider: fireworks + model_id: accounts/fireworks/models/deepseek-v4-flash + + # --- OpenRouter: "none" truly disables; grok/kimi need special toggles --- + inkling-small: + provider: openrouter + model_id: thinkingmachines/inkling-small + capabilities: + reasoning: { extra: { toggle: effort }, off_value: none, on_value: high, true_off: true } + + grok-4.3: + provider: openrouter + model_id: x-ai/grok-4.3 + capabilities: + # Grok rejects reasoning.enabled=false ("Reasoning is mandatory"); floor is effort=minimal. + reasoning: { extra: { toggle: effort }, off_value: minimal, on_value: high, true_off: false } + + kimi-k2.6: + provider: openrouter + model_id: moonshotai/kimi-k2.6 + capabilities: + # Pin to Moonshot's own inference; disable via the enabled toggle. + reasoning: + extra: { toggle: enabled, provider_only: [moonshotai] } + off_value: false + on_value: true + true_off: true + + # --- Gemini: 3.x uses thinking_level (floor "low"); 2.5 uses integer budget --- + gemini-3.7-flash: + provider: gemini + model_id: gemini-3.7-flash + + gemini-2.5-flash: + provider: gemini + model_id: gemini-2.5-flash + capabilities: + reasoning: { style: thinking_budget, off_value: 0, on_value: -1, true_off: true } + + # --- OpenAI / Anthropic / Interfaze: provider defaults fit as-is --- + gpt-5.5: + provider: openai + model_id: gpt-5.5 + + claude-sonnet-4-6: + provider: anthropic + model_id: claude-sonnet-4-6 + + interfaze-beta: + provider: interfaze + model_id: interfaze-beta diff --git a/bench_core/targets/claude-sonnet-4-6.yaml b/bench_core/targets/claude-sonnet-4-6.yaml deleted file mode 100644 index 72e6f9c..0000000 --- a/bench_core/targets/claude-sonnet-4-6.yaml +++ /dev/null @@ -1,5 +0,0 @@ -name: claude-sonnet-4-6 -provider: anthropic -model_id: claude-sonnet-4-6 -# Anthropic defaults fit: thinking disabled/enabled block, temperature omitted -# when thinking is on, max_tokens forced above budget_tokens. diff --git a/bench_core/targets/gemini-2.5-flash.yaml b/bench_core/targets/gemini-2.5-flash.yaml deleted file mode 100644 index 117815b..0000000 --- a/bench_core/targets/gemini-2.5-flash.yaml +++ /dev/null @@ -1,11 +0,0 @@ -name: gemini-2.5-flash -provider: gemini -model_id: gemini-2.5-flash -capabilities: - # 2.5 uses the older integer thinking_budget, not thinking_level. - # 0 is a genuine disable on 2.5-flash; -1 is dynamic (full thinking). - reasoning: - style: thinking_budget - off_value: 0 - on_value: -1 - true_off: true diff --git a/bench_core/targets/gemini-3.7-flash.yaml b/bench_core/targets/gemini-3.7-flash.yaml deleted file mode 100644 index b66425f..0000000 --- a/bench_core/targets/gemini-3.7-flash.yaml +++ /dev/null @@ -1,5 +0,0 @@ -name: gemini-3.7-flash -provider: gemini -model_id: gemini-3.7-flash -# Gemini defaults fit: thinking_level with "low" as the floor (3.7 flash rejects -# "minimal"), image/audio via native Parts. diff --git a/bench_core/targets/gpt-5.5.yaml b/bench_core/targets/gpt-5.5.yaml deleted file mode 100644 index 8798d13..0000000 --- a/bench_core/targets/gpt-5.5.yaml +++ /dev/null @@ -1,5 +0,0 @@ -name: gpt-5.5 -provider: openai -model_id: gpt-5.5 -# OpenAI defaults fit: reasoning_effort with "none" a true off; temperature is -# omitted automatically when reasoning is engaged. diff --git a/bench_core/targets/grok-4.3.yaml b/bench_core/targets/grok-4.3.yaml deleted file mode 100644 index 7720fc0..0000000 --- a/bench_core/targets/grok-4.3.yaml +++ /dev/null @@ -1,11 +0,0 @@ -name: grok-4.3 -provider: openrouter -model_id: x-ai/grok-4.3 -capabilities: - # Grok's OpenRouter endpoint rejects reasoning.enabled=false ("Reasoning is - # mandatory"), so use the effort toggle with "minimal" as the floor. - reasoning: - extra: { toggle: effort } - off_value: minimal - on_value: high - true_off: false diff --git a/bench_core/targets/inkling-small.yaml b/bench_core/targets/inkling-small.yaml deleted file mode 100644 index 4fbff65..0000000 --- a/bench_core/targets/inkling-small.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: inkling-small -provider: openrouter -model_id: thinkingmachines/inkling-small -capabilities: - # OpenRouter "none" truly disables reasoning here (unlike the Fireworks floor). - reasoning: - extra: { toggle: effort } - off_value: none - on_value: high - true_off: true diff --git a/bench_core/targets/inkling.yaml b/bench_core/targets/inkling.yaml deleted file mode 100644 index 21f8e17..0000000 --- a/bench_core/targets/inkling.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# A benchmark TARGET: everything needed to run any benchmark against one model. -# Declare only what differs from the provider's defaults (see bench_core/config.py -# PROVIDERS). Fireworks defaults already encode Inkling's traits: reasoning -# effort with "none" as a FLOOR (still thinks), audio via audio_url, temperature -# sent regardless. So nothing else is needed here. -name: inkling -provider: fireworks -model_id: accounts/fireworks/models/inkling -# ci_regression: true # opt into the nightly regression matrix diff --git a/bench_core/targets/interfaze-beta.yaml b/bench_core/targets/interfaze-beta.yaml deleted file mode 100644 index d3b47bc..0000000 --- a/bench_core/targets/interfaze-beta.yaml +++ /dev/null @@ -1,5 +0,0 @@ -name: interfaze-beta -provider: interfaze -model_id: interfaze-beta -# Interfaze defaults fit: reasoning_effort vocab is off/minimal/low/.../high/on/auto -# (NOT "none"); audio via the OpenAI "file" block; base_url from OPENAI_BASE_URL. diff --git a/bench_core/targets/kimi-k2.6.yaml b/bench_core/targets/kimi-k2.6.yaml deleted file mode 100644 index c58fe66..0000000 --- a/bench_core/targets/kimi-k2.6.yaml +++ /dev/null @@ -1,13 +0,0 @@ -name: kimi-k2.6 -provider: openrouter -model_id: moonshotai/kimi-k2.6 -capabilities: - # Pin to Moonshot's own inference (avoid reseller deployments) and disable - # reasoning via the enabled toggle. - reasoning: - extra: - toggle: enabled - provider_only: [moonshotai] - off_value: false - on_value: true - true_off: true diff --git a/tests/test_config.py b/tests/test_config.py index 27903b5..916e2e4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,52 +1,86 @@ -"""Target loading + the capability merge chain. A target YAML declares only what -differs from its provider's defaults; per-benchmark overrides and CLI overrides -layer on top. build_adapter maps the provider to the right adapter class. +"""Target loading + the capability merge chain. Targets live in one file under a +`targets:` map (key = name); a target declares only what differs from its +provider's defaults, and per-benchmark / CLI overrides layer on top. +build_adapter maps the provider to the right adapter class. """ import textwrap from bench_core.capabilities import AudioShape, ReasoningStyle -from bench_core.config import build_adapter, load_target, resolve_capabilities +from bench_core.config import ( + build_adapter, + load_all_targets, + load_target, + resolve_capabilities, +) from bench_core.providers.gemini import GeminiAdapter from bench_core.providers.openai_compat import OpenAICompatAdapter -def _write(tmp_path, text): - p = tmp_path / "target.yaml" - p.write_text(textwrap.dedent(text)) +def _targets(tmp_path, body): + p = tmp_path / "targets.yaml" + p.write_text("targets:\n" + textwrap.indent(textwrap.dedent(body), " ")) return p def test_load_target_parses_core_fields(tmp_path): - t = load_target( - _write( - tmp_path, - """ - name: inkling - provider: fireworks - model_id: accounts/fireworks/models/inkling + path = _targets( + tmp_path, + """ + inkling: + provider: fireworks + model_id: accounts/fireworks/models/inkling """, - ) ) + t = load_target("inkling", path) assert t.name == "inkling" assert t.provider == "fireworks" assert t.model_id == "accounts/fireworks/models/inkling" +def test_load_all_targets_returns_every_entry(tmp_path): + path = _targets( + tmp_path, + """ + a: + provider: fireworks + model_id: x + b: + provider: gemini + model_id: y + """, + ) + targets = load_all_targets(path) + assert set(targets) == {"a", "b"} + + +def test_unknown_target_name_raises(tmp_path): + import pytest + + path = _targets( + tmp_path, + """ + a: + provider: fireworks + model_id: x + """, + ) + with pytest.raises(ValueError, match="unknown target"): + load_target("nope", path) + + def test_target_inherits_provider_defaults(tmp_path): # A fireworks target declares no media; it should inherit audio_url + the # "none is a floor" reasoning semantics from the provider defaults. - t = load_target( - _write( - tmp_path, - """ - name: inkling - provider: fireworks - model_id: x + path = _targets( + tmp_path, + """ + inkling: + provider: fireworks + model_id: x """, - ) ) - caps = resolve_capabilities(t) + caps = resolve_capabilities(load_target("inkling", path)) assert caps.media.audio is AudioShape.AUDIO_URL assert caps.reasoning.style is ReasoningStyle.EFFORT assert caps.reasoning.true_off is False # Fireworks "none" still thinks @@ -54,68 +88,73 @@ def test_target_inherits_provider_defaults(tmp_path): def test_target_overrides_win_over_defaults(tmp_path): # gemini-2.5-flash uses the budget knob, overriding the thinking_level default. - t = load_target( - _write( - tmp_path, - """ - name: gemini-2.5-flash - provider: gemini - model_id: gemini-2.5-flash - capabilities: - reasoning: - style: thinking_budget - off_value: 0 - on_value: -1 - true_off: true + path = _targets( + tmp_path, + """ + gemini-2.5-flash: + provider: gemini + model_id: gemini-2.5-flash + capabilities: + reasoning: + style: thinking_budget + off_value: 0 + on_value: -1 + true_off: true """, - ) ) - caps = resolve_capabilities(t) + caps = resolve_capabilities(load_target("gemini-2.5-flash", path)) assert caps.reasoning.style is ReasoningStyle.THINKING_BUDGET assert caps.reasoning.off_value == 0 def test_per_benchmark_override_applies(tmp_path): - t = load_target( - _write( - tmp_path, - """ - name: gemini-3.7-flash - provider: gemini - model_id: gemini-3.7-flash - capabilities: - reasoning: { style: thinking_level, off_value: low, on_value: high, true_off: false } - overrides: - asr: - reasoning: { off_value: high } + path = _targets( + tmp_path, + """ + gemini-3.7-flash: + provider: gemini + model_id: gemini-3.7-flash + capabilities: + reasoning: { style: thinking_level, off_value: low, on_value: high, true_off: false } + overrides: + asr: + reasoning: { off_value: high } """, - ) ) + t = load_target("gemini-3.7-flash", path) assert resolve_capabilities(t, benchmark="gpqa").reasoning.off_value == "low" assert resolve_capabilities(t, benchmark="asr").reasoning.off_value == "high" def test_cli_override_wins_over_everything(tmp_path): - t = load_target( - _write( - tmp_path, - """ - name: inkling - provider: fireworks - model_id: x + path = _targets( + tmp_path, + """ + inkling: + provider: fireworks + model_id: x """, - ) ) - caps = resolve_capabilities(t, cli_overrides={"reasoning": {"on_value": "max"}}) + caps = resolve_capabilities( + load_target("inkling", path), cli_overrides={"reasoning": {"on_value": "max"}} + ) assert caps.reasoning.on_value == "max" def test_build_adapter_maps_provider_to_class(tmp_path): - fw = load_target(_write(tmp_path, "name: a\nprovider: fireworks\nmodel_id: x\n")) - a = build_adapter(fw) + path = _targets( + tmp_path, + """ + a: + provider: fireworks + model_id: x + b: + provider: gemini + model_id: y + """, + ) + a = build_adapter(load_target("a", path)) assert isinstance(a, OpenAICompatAdapter) assert "fireworks_api_key" in [k.lower() for k in a.key_spec] assert a.base_url and "fireworks" in a.base_url - - gtarget = load_target(_write(tmp_path, "name: b\nprovider: gemini\nmodel_id: y\n")) - assert isinstance(build_adapter(gtarget), GeminiAdapter) + assert isinstance(build_adapter(load_target("b", path)), GeminiAdapter) diff --git a/tests/test_targets.py b/tests/test_targets.py index 369de0a..2e88dea 100644 --- a/tests/test_targets.py +++ b/tests/test_targets.py @@ -1,45 +1,39 @@ -"""Every shipped target YAML must load, resolve to typed capabilities, and build -its adapter — with no network. This is the guard behind the "add a YAML, no -code" promise: a new target file is validated here (and in CI on push) rather -than failing mid-run. +"""Every shipped target must load, resolve to typed capabilities, and build its +adapter — with no network. This is the guard behind the "add a target = add a +YAML entry, no code" promise: a new entry is validated here (and in CI on push) +rather than failing mid-run. """ -from pathlib import Path - import pytest from bench_core.capabilities import Capabilities -from bench_core.config import build_adapter, load_target, resolve_capabilities +from bench_core.config import build_adapter, load_all_targets, resolve_capabilities -TARGET_DIR = Path(__file__).resolve().parent.parent / "bench_core" / "targets" -TARGET_FILES = sorted(TARGET_DIR.glob("*.yaml")) +TARGETS = load_all_targets() +TARGET_NAMES = sorted(TARGETS) def test_targets_exist(): - assert TARGET_FILES, "no target YAML files found" + assert TARGET_NAMES, "no targets defined in bench_core/targets.yaml" -@pytest.mark.parametrize("path", TARGET_FILES, ids=lambda p: p.stem) -def test_target_loads_resolves_and_builds_adapter(path): - target = load_target(path) - # the filename should match the declared name (keeps the dispatch dropdown honest) - assert target.name == path.stem +@pytest.mark.parametrize("name", TARGET_NAMES) +def test_target_resolves_and_builds_adapter(name): + target = TARGETS[name] caps = resolve_capabilities(target) assert isinstance(caps, Capabilities) - # per-benchmark override paths must also resolve - resolve_capabilities(target, benchmark="asr") + resolve_capabilities(target, benchmark="asr") # per-benchmark path must resolve too adapter = build_adapter(target) assert adapter.name == target.provider - assert adapter.key_spec # some env var to read a key from + assert adapter.key_spec -@pytest.mark.parametrize("path", TARGET_FILES, ids=lambda p: p.stem) -def test_reasoning_off_and_on_values_are_not_yaml_booleans_by_accident(path): - # Guards the off:/on: -> bool trap: if a future edit uses bare off:/on: keys, - # the reasoning dict loses its values and this catches it. - target = load_target(path) - reasoning = target.capabilities.get("reasoning", {}) +@pytest.mark.parametrize("name", TARGET_NAMES) +def test_reasoning_keys_are_not_yaml_booleans_by_accident(name): + # Guards the off:/on: -> bool trap: bare off:/on: keys parse as booleans and + # silently drop the reasoning values. + reasoning = TARGETS[name].capabilities.get("reasoning", {}) assert False not in reasoning and True not in reasoning, ( - f"{path.name} used bare off:/on: keys (parsed as booleans) — " + f"target {name!r} used bare off:/on: keys (parsed as booleans) — " "use off_value:/on_value:" ) From 2c12cc7c4f4a437b43cb4cdae441239c58393b7c Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 07:43:14 +0530 Subject: [PATCH 04/38] chore(bench_core): add anthropic dependency; remove dead line in openai_compat anthropic>=0.122.0 is required by AnthropicAdapter.build_client (was uncommitted). Also drops an unreachable duplicate return an automated pass left in openai_compat._content. --- bench_core/providers/anthropic.py | 10 +++------ bench_core/providers/base.py | 8 ------- bench_core/providers/gemini.py | 8 +++---- bench_core/providers/openai_compat.py | 9 +++----- pyproject.toml | 1 + uv.lock | 30 +++++++++++++++++++++++++++ 6 files changed, 40 insertions(+), 26 deletions(-) diff --git a/bench_core/providers/anthropic.py b/bench_core/providers/anthropic.py index 71775ae..012a530 100644 --- a/bench_core/providers/anthropic.py +++ b/bench_core/providers/anthropic.py @@ -1,8 +1,3 @@ -"""Anthropic adapter. System messages become the top-level `system` param; -`max_tokens` is mandatory and must exceed `budget_tokens` when thinking is -enabled — both handled declaratively from the cap. -""" - from __future__ import annotations from bench_core.capabilities import Capabilities @@ -32,7 +27,8 @@ def build_client(self, api_key: str | None = None): def _content(self, msg: Message, caps: Capabilities): if all(isinstance(p, TextPart) for p in msg.parts): - return "".join(p.text for p in msg.parts) + text_parts = [p for p in msg.parts if isinstance(p, TextPart)] + return "".join(p.text for p in text_parts) blocks = [] for p in msg.parts: if isinstance(p, TextPart): @@ -65,7 +61,7 @@ def encode(self, req: Request, caps: Capabilities, model_id: str) -> dict: thinking_on = (inj.anthropic_thinking or {}).get("type") == "enabled" if thinking_on: - budget = inj.anthropic_thinking["budget_tokens"] + budget = inj.anthropic_thinking.get("budget_tokens", 0) if inj.anthropic_thinking else 0 max_tokens = req.max_tokens or extra.get("on_max_tokens", _DEFAULT_ON_MAX) if max_tokens <= budget: max_tokens = budget + _BUDGET_MARGIN diff --git a/bench_core/providers/base.py b/bench_core/providers/base.py index 370e5e4..0952a0d 100644 --- a/bench_core/providers/base.py +++ b/bench_core/providers/base.py @@ -1,11 +1,3 @@ -"""Provider adapter contract + shared behavior. - -`decode` and `classify_error` are identical across hosts (they dispatch on the -capability's response shape / the exception), so they live here. Subclasses -supply `build_client`, `encode`, and `call` — the parts that are genuinely -provider-specific. -""" - from __future__ import annotations import os diff --git a/bench_core/providers/gemini.py b/bench_core/providers/gemini.py index f5fcb17..81ef44c 100644 --- a/bench_core/providers/gemini.py +++ b/bench_core/providers/gemini.py @@ -1,7 +1,5 @@ -"""Gemini (native google-genai) adapter. Reasoning becomes a ThinkingConfig -(level for 3.x, budget for 2.5); media parts become genai Parts. The client is -cached at the factory level to avoid the "client has been closed" GC bug. -""" +# Reasoning becomes a ThinkingConfig (level for 3.x, budget for 2.5); media parts become genai Parts. +# The client is cached at the factory level to avoid the "client has been closed" GC bug. from __future__ import annotations @@ -50,7 +48,7 @@ def encode(self, req: Request, caps: Capabilities, model_id: str) -> dict: inj = build_reasoning(req.reasoning.mode, caps.reasoning) thinking = None if inj.thinking_level is not None: - thinking = types.ThinkingConfig(thinking_level=inj.thinking_level) + thinking = types.ThinkingConfig(thinking_level=types.ThinkingLevel(inj.thinking_level)) elif inj.thinking_budget is not None: thinking = types.ThinkingConfig(thinking_budget=inj.thinking_budget) diff --git a/bench_core/providers/openai_compat.py b/bench_core/providers/openai_compat.py index 73c7c17..029d276 100644 --- a/bench_core/providers/openai_compat.py +++ b/bench_core/providers/openai_compat.py @@ -1,8 +1,4 @@ -"""Adapter for hosts that speak OpenAI Chat Completions: openai, fireworks, -openrouter, interfaze. One `encode`; the four concrete providers differ only in -base_url / key_spec / capability defaults (defined in their target YAML + a thin -factory), not in code. -""" +# OpenAI Chat Completions: openai, fireworks, openrouter, interfaze. from __future__ import annotations @@ -34,7 +30,8 @@ def build_client(self, api_key: str | None = None): def _content(self, msg: Message, caps: Capabilities) -> Any: if all(isinstance(p, TextPart) for p in msg.parts): - return "".join(p.text for p in msg.parts) + text_parts = [p for p in msg.parts if isinstance(p, TextPart)] + return "".join(p.text for p in text_parts) blocks = [] for p in msg.parts: if isinstance(p, TextPart): diff --git a/pyproject.toml b/pyproject.toml index 7a0bf4f..aba7e9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "playwright>=1.44.0", "gdown>=6.1.0", "google-genai>=2.18.1", + "anthropic>=0.122.0", ] [dependency-groups] diff --git a/uv.lock b/uv.lock index 5767660..b3aa3d5 100644 --- a/uv.lock +++ b/uv.lock @@ -144,6 +144,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, ] +[[package]] +name = "anthropic" +version = "0.122.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "docstring-parser" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/23/9987d70b74e3481d5bc5d2021d3e10fd5f60c1f7b54088ea86506d9b7f2b/anthropic-0.122.0.tar.gz", hash = "sha256:ffec56ae96657c8d19fa575ec96f140f380c353a07ab7d61b92eb18ee6536601", size = 1021535, upload-time = "2026-08-13T18:36:00.307Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/5b/f5c87e71097a9f89f1b414d1ef7ae8439051fae57d5e4ee90946082982b8/anthropic-0.122.0-py3-none-any.whl", hash = "sha256:45ec906452ffae6b5f7f0c53d01f50bfb7e4ce878d7ae8e4309d13171e557e67", size = 1041853, upload-time = "2026-08-13T18:36:01.831Z" }, +] + [[package]] name = "anyio" version = "4.14.2" @@ -623,6 +642,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] +[[package]] +name = "docstring-parser" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, +] + [[package]] name = "editdistance" version = "0.8.1" @@ -1203,6 +1231,7 @@ name = "interfaze-complete-benchmarks" version = "0.1.2" source = { virtual = "." } dependencies = [ + { name = "anthropic" }, { name = "apted" }, { name = "beautifulsoup4" }, { name = "datasets" }, @@ -1252,6 +1281,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "anthropic", specifier = ">=0.122.0" }, { name = "apted", specifier = ">=1.0.3" }, { name = "beautifulsoup4", specifier = ">=4.11.0" }, { name = "datasets", specifier = ">=4.7.0" }, From 8f563e088736270440e8e26399b01aed6210b85f Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 08:11:10 +0530 Subject: [PATCH 05/38] docs(bench_core): trim module docstrings to one line Keep only load-bearing comments; drop multi-paragraph module narration. --- bench_core/capabilities.py | 9 +-------- bench_core/config.py | 10 +--------- bench_core/decode.py | 5 +---- bench_core/errors.py | 8 +------- bench_core/execute.py | 10 +--------- bench_core/media.py | 6 +----- bench_core/reasoning.py | 10 +--------- bench_core/request.py | 5 +---- bench_core/response.py | 4 +--- bench_core/results.py | 10 +--------- bench_core/runner.py | 6 +----- bench_core/targets.yaml | 19 ------------------- 12 files changed, 11 insertions(+), 91 deletions(-) diff --git a/bench_core/capabilities.py b/bench_core/capabilities.py index 23530b6..fc62af7 100644 --- a/bench_core/capabilities.py +++ b/bench_core/capabilities.py @@ -1,11 +1,4 @@ -"""Declared model capabilities + the merge that resolves them. - -Every axis the audit found varying across providers/models is a field here, so -that a new model is described by *data* (a target YAML file) rather than a new -code path. Provider adapters ship a default capability dict; the target file -overrides fields; the CLI can override again. `deep_merge` collapses the chain; -`Capabilities.from_dict` gives adapters a typed, validated view. -""" +"""Declared model capabilities + the merge that resolves them.""" from __future__ import annotations diff --git a/bench_core/config.py b/bench_core/config.py index 3e65e2a..ec99c09 100644 --- a/bench_core/config.py +++ b/bench_core/config.py @@ -1,12 +1,4 @@ -"""Targets + provider registry + the capability merge chain. - -A provider ships defaults (base_url, key precedence, and the host's baseline -capabilities); a target YAML declares only what differs; per-benchmark and CLI -overrides layer on top. `resolve_capabilities` collapses the chain into a typed -`Capabilities`; `build_adapter` maps the provider name to its adapter class. - -Adding a MODEL is a YAML file. Adding a PROVIDER is one registry entry here. -""" +"""Targets + provider registry + the capability merge chain.""" from __future__ import annotations diff --git a/bench_core/decode.py b/bench_core/decode.py index 3af19be..cb1c4f9 100644 --- a/bench_core/decode.py +++ b/bench_core/decode.py @@ -1,7 +1,4 @@ -"""Normalize a host response into `Response`. One function per response shape; -the reasoning-token fallbacks (Fireworks top-level, Gemini thoughts count) live -here instead of in every runner. -""" +"""Normalize a host response into `Response` (unifies the three shapes).""" from __future__ import annotations diff --git a/bench_core/errors.py b/bench_core/errors.py index de10871..2301824 100644 --- a/bench_core/errors.py +++ b/bench_core/errors.py @@ -1,10 +1,4 @@ -"""Classify a provider exception into the signal the fallback ladder acts on. - -Status code comes from the SDK attribute when present, else the leading digits -of the message (google-genai raises a plain exception whose text starts with -"400 INVALID_ARGUMENT ..."). PARAM_REJECTED carries an `action` the ladder maps -to a request mutation, so a genuinely new model degrades instead of crashing. -""" +"""Classify a provider exception into the signal the fallback ladder acts on.""" from __future__ import annotations diff --git a/bench_core/execute.py b/bench_core/execute.py index 0dde322..ac64a85 100644 --- a/bench_core/execute.py +++ b/bench_core/execute.py @@ -1,12 +1,4 @@ -"""The fallback ladder — one request through an adapter, self-healing the -declared-capability gaps a genuinely new model exposes. - -`PARAM_REJECTED` is handled here: mutate the request/caps per the classified -action, record a capability_hint (so the user promotes the fix into the target -YAML), and retry immediately. Everything else (rate-limit, transient, empty, -fatal) is raised as `BenchError` carrying the classification, for the runner to -retry-with-backoff or fail. -""" +"""The fallback ladder: one request through an adapter, self-healing declared-cap gaps.""" from __future__ import annotations diff --git a/bench_core/media.py b/bench_core/media.py index 3b326d4..f004763 100644 --- a/bench_core/media.py +++ b/bench_core/media.py @@ -1,8 +1,4 @@ -"""Media block builders — one per host shape, so the benchmark never picks a -shape and the silent-drop look-alikes (audio_url vs input_audio) can't be -confused. `encode_image` is the single place RGB-convert + resize + encode -happens (the audit found it in only 1 of 9 OCRBench variants). -""" +"""Media block builders — one per host shape (image + audio).""" from __future__ import annotations diff --git a/bench_core/reasoning.py b/bench_core/reasoning.py index 791f643..1e21b34 100644 --- a/bench_core/reasoning.py +++ b/bench_core/reasoning.py @@ -1,12 +1,4 @@ -"""Translate an abstract reasoning *mode* into the concrete directive a host -expects — the one place the audit's six divergent Gemini-floor encodings (plus -the OpenAI/Fireworks/Anthropic/OpenRouter variants) now live. - -A benchmark asks for a mode ("off" / "low" / "medium" / "high"); the model's -declared `ReasoningCap` says how that host expresses it and what its floor is. -`build_reasoning` returns a `ReasoningInjection` describing where the directive -goes; each provider adapter reads only the field for its mechanism. -""" +"""Translate an abstract reasoning mode into the host's concrete directive.""" from __future__ import annotations diff --git a/bench_core/request.py b/bench_core/request.py index 26a9cbe..29ede2b 100644 --- a/bench_core/request.py +++ b/bench_core/request.py @@ -1,7 +1,4 @@ -"""Provider-agnostic request the benchmark layer builds. Benchmarks construct -these and never name a provider; the adapter's `encode` turns them into the -host-native payload. -""" +"""Provider-agnostic request the benchmark layer builds.""" from __future__ import annotations diff --git a/bench_core/response.py b/bench_core/response.py index d691f7e..43dfa96 100644 --- a/bench_core/response.py +++ b/bench_core/response.py @@ -1,6 +1,4 @@ -"""Normalized response — one shape regardless of host, so benchmark `parse` -never touches choices[0] / content-blocks / .text directly. -""" +"""Normalized response — one shape regardless of host.""" from __future__ import annotations diff --git a/bench_core/results.py b/bench_core/results.py index 4df36a2..6788bb2 100644 --- a/bench_core/results.py +++ b/bench_core/results.py @@ -1,12 +1,4 @@ -"""The single result contract. One slug, one directory layout, one discovery -path — so a run can't be silently dropped by a prefix mismatch and two models -can't overwrite each other's checkpoint. - - results/// - responses.jsonl # resume source, one row per sample - metrics.json # canonical schema (see cli/benchmark code) - run.json # config snapshot -""" +"""The single result contract: one slug, one layout, one content-based discovery.""" from __future__ import annotations diff --git a/bench_core/runner.py b/bench_core/runner.py index 61d1d16..b8cd286 100644 --- a/bench_core/runner.py +++ b/bench_core/runner.py @@ -1,8 +1,4 @@ -"""The one execution harness. Owns resume, bounded concurrency, rate limiting, -retry-with-backoff, and incremental checkpointing so no benchmark reimplements -them (the audit found ~15 divergent copies). Benchmarks supply only -`build_request` and `parse`; inference goes through the fallback ladder. -""" +"""The one execution harness: rate-limit, semaphore, retry, resume, checkpoint.""" from __future__ import annotations diff --git a/bench_core/targets.yaml b/bench_core/targets.yaml index f322581..3028c7a 100644 --- a/bench_core/targets.yaml +++ b/bench_core/targets.yaml @@ -1,22 +1,3 @@ -# All benchmark targets in one file. The map KEY is the target name (also the -# result-directory name). Declare only what differs from the provider's defaults -# (see bench_core/config.py PROVIDERS). Adding a model = adding an entry here; no -# code change. -# -# Schema per target: -# provider: fireworks | openrouter | openai | anthropic | gemini | interfaze -# model_id: the id the provider expects -# capabilities: (optional) overrides of the provider defaults -# reasoning: { style, off_value, on_value, true_off, temperature_when_on, extra } -# media: { image, audio } -# response: openai_chat | anthropic_blocks | gemini_text -# overrides: (optional) per-benchmark capability tweaks, e.g. { asr: {...} } -# harness_specific: (optional) opaque config for isolated harnesses, e.g. { bfcl: {...} } -# ci_regression: true (optional) include in the nightly regression matrix -# -# NB: reasoning keys are off_value/on_value, NOT off/on — bare `off:`/`on:` are -# YAML booleans (off: low -> {False: low}). - targets: # --- Fireworks: reasoning "none" is a FLOOR (still thinks); audio via audio_url --- inkling: From 4b710e30f7e57e0d8ea7e748ef3970bc3901d8f2 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 08:13:08 +0530 Subject: [PATCH 06/38] chore: remove one-off debug probe scripts _probe_voxpopuli_missing.py and _probe_grok_reasoning.py were diagnostic one-offs (unreferenced). Unrelated to the runners. --- benchmarks/asr/_probe_voxpopuli_missing.py | 103 ------------------ .../obj_detection/_probe_grok_reasoning.py | 52 --------- 2 files changed, 155 deletions(-) delete mode 100644 benchmarks/asr/_probe_voxpopuli_missing.py delete mode 100644 benchmarks/obj_detection/_probe_grok_reasoning.py diff --git a/benchmarks/asr/_probe_voxpopuli_missing.py b/benchmarks/asr/_probe_voxpopuli_missing.py deleted file mode 100644 index b1d5969..0000000 --- a/benchmarks/asr/_probe_voxpopuli_missing.py +++ /dev/null @@ -1,103 +0,0 @@ -import sys -from pathlib import Path - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -from datasets import load_dataset -from google import genai -from google.genai import types - -from benchmarks.asr.voxpopuli_aa import ( - DATASET_ID, - PROMPT, - SPLIT, - build_sample, - fetch_audio_bytes, -) -from benchmarks.asr.voxpopuli_aa_multi import _load_interfaze_env - -TARGET_ID = "20150527-0900-PLENARY-14-en_20150527-21:41:31_1" -MODELS = ["gemini-2.5-pro", "gemini-3-flash-preview", "gemini-3.1-pro-preview"] - - -def thinking_for(model: str) -> types.ThinkingConfig: - m = model.lower() - if m.startswith("gemini-2.5-pro"): - return types.ThinkingConfig(thinking_budget=128) - if m.startswith("gemini-2.5-flash"): - return types.ThinkingConfig(thinking_budget=0) - if "pro" in m: - return types.ThinkingConfig(thinking_level="low") - return types.ThinkingConfig(thinking_level="minimal") - - -def main(): - env = _load_interfaze_env() - client = genai.Client(api_key=env["GEMINI_KEY"]) - - print(f"Loading dataset {DATASET_ID} split={SPLIT}...") - ds = load_dataset(DATASET_ID, split=SPLIT) - sample = None - for row in ds: - s = build_sample(dict(row)) - if s["id"] == TARGET_ID: - sample = s - break - if sample is None: - print(f"!! sample {TARGET_ID} not found in dataset"); sys.exit(1) - - print(f"\nSample: id={sample['id']} dur={sample['duration']}s lang={sample['language']}") - print(f"GT transcript: {sample['transcript']!r}\n") - - audio_bytes = fetch_audio_bytes(sample["file_name"]) - print(f"Audio fetched: {len(audio_bytes)} bytes\n") - - for model in MODELS: - print("=" * 80) - print(f"MODEL: {model}") - print("=" * 80) - config = types.GenerateContentConfig( - thinking_config=thinking_for(model), - temperature=0.0, - ) - try: - resp = client.models.generate_content( - model=model, - contents=[ - types.Part.from_bytes(data=audio_bytes, mime_type="audio/wav"), - PROMPT, - ], - config=config, - ) - except Exception as e: - print(f" EXCEPTION: {type(e).__name__}: {e}\n") - continue - - text = (resp.text or "").strip() if hasattr(resp, "text") else "" - print(f" resp.text : {text!r}") - print(f" response_id : {getattr(resp, 'response_id', None)}") - - pf = getattr(resp, "prompt_feedback", None) - print(f" prompt_feedback : {pf}") - - cands = getattr(resp, "candidates", None) or [] - print(f" num candidates : {len(cands)}") - for i, c in enumerate(cands): - fr = getattr(c, "finish_reason", None) - fm = getattr(c, "finish_message", None) - sr = getattr(c, "safety_ratings", None) - content = getattr(c, "content", None) - parts = getattr(content, "parts", None) if content else None - part_texts = [getattr(p, "text", None) for p in (parts or [])] - print(f" candidate[{i}].finish_reason : {fr}") - print(f" candidate[{i}].finish_message : {fm}") - print(f" candidate[{i}].safety_ratings : {sr}") - print(f" candidate[{i}].part texts : {part_texts}") - - usage = getattr(resp, "usage_metadata", None) - print(f" usage_metadata : {usage}\n") - - -if __name__ == "__main__": - main() diff --git a/benchmarks/obj_detection/_probe_grok_reasoning.py b/benchmarks/obj_detection/_probe_grok_reasoning.py deleted file mode 100644 index cb20f4f..0000000 --- a/benchmarks/obj_detection/_probe_grok_reasoning.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Probe what reasoning settings x-ai/grok-4.3 accepts on OpenRouter.""" -import os -import sys -import json -from pathlib import Path - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -from dotenv import load_dotenv -from openai import OpenAI - -load_dotenv() - -key = os.getenv("OPENROUTER_API_KEY") or os.getenv("OPENROUTER_KEY") -client = OpenAI(base_url="https://openrouter.ai/api/v1", api_key=key) -MODEL = "x-ai/grok-4.3" -PROMPT = "What is 2+2? Reply with only the number." - -trials = [ - ("no_reasoning_field", {}), - ("reasoning.enabled=False", {"reasoning": {"enabled": False}}), - ("reasoning.enabled=True", {"reasoning": {"enabled": True}}), - ("reasoning.effort=minimal", {"reasoning": {"effort": "minimal"}}), - ("reasoning.effort=low", {"reasoning": {"effort": "low"}}), - ("reasoning.effort=medium", {"reasoning": {"effort": "medium"}}), - ("reasoning.effort=high", {"reasoning": {"effort": "high"}}), - ("reasoning.max_tokens=1", {"reasoning": {"max_tokens": 1}}), - ("reasoning.max_tokens=128", {"reasoning": {"max_tokens": 128}}), -] - -for label, extra in trials: - print("=" * 70) - print(f"trial: {label} body={extra}") - try: - r = client.chat.completions.create( - model=MODEL, - messages=[{"role": "user", "content": PROMPT}], - temperature=0.0, - extra_body=extra, - ) - msg = r.choices[0].message - content = (getattr(msg, "content", None) or "").strip() - reasoning = getattr(msg, "reasoning", None) - usage = getattr(r, "usage", None) - print(f" OK content={content!r}") - print(f" reasoning preview: {(reasoning or '')[:120]!r}") - if usage is not None: - ud = usage.model_dump() if hasattr(usage, "model_dump") else dict(usage) - print(f" usage: {json.dumps(ud, default=str)}") - except Exception as e: - print(f" ERROR {type(e).__name__}: {e}") From 4ba6054fc1ab728bfa478887beb637006caf7ab9 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 08:21:36 +0530 Subject: [PATCH 07/38] feat(bench_core): migrate GPQA onto the abstraction; retire 4 per-provider forks Adds benchmarks/gpqa/bench.py (dataset + prompt + parse + score), the bench CLI (run/list-targets) + python -m bench_core entrypoint, and an offline scoring-parity test that replays all 4 archived GPQA runs through the new parse/score and asserts bit-identical scores (inkling 87.37, deepseek 54.55, gemini 90.40, inkling-small 85.86). Deletes gpqa_{openai,openrouter,fireworks,gemini}.py (4 forks -> 1 bench.py). Updates run_fireworks_suite.sh (gpqa via CLI, TARGET env) and README. --- README.md | 27 +- bench_core/__main__.py | 3 + bench_core/cli.py | 132 ++++++++ benchmarks/gpqa/bench.py | 119 ++++++++ benchmarks/gpqa/gpqa_fireworks.py | 324 -------------------- benchmarks/gpqa/gpqa_gemini.py | 277 ----------------- benchmarks/gpqa/gpqa_openai.py | 467 ----------------------------- benchmarks/gpqa/gpqa_openrouter.py | 276 ----------------- scripts/run_fireworks_suite.sh | 12 +- tests/test_gpqa_bench.py | 85 ++++++ 10 files changed, 353 insertions(+), 1369 deletions(-) create mode 100644 bench_core/__main__.py create mode 100644 bench_core/cli.py create mode 100644 benchmarks/gpqa/bench.py delete mode 100644 benchmarks/gpqa/gpqa_fireworks.py delete mode 100644 benchmarks/gpqa/gpqa_gemini.py delete mode 100644 benchmarks/gpqa/gpqa_openai.py delete mode 100644 benchmarks/gpqa/gpqa_openrouter.py create mode 100644 tests/test_gpqa_bench.py diff --git a/README.md b/README.md index eb8f6b2..3a6eaa8 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ audio input, so it runs on every benchmark here: ```bash uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks uv run -m benchmarks.olmocr.olmocr_bench_fireworks -uv run -m benchmarks.gpqa.gpqa_fireworks +uv run python -m bench_core run --target inkling --benchmark gpqa uv run -m benchmarks.mmmlu.mmmlu_multi --provider fireworks --model accounts/fireworks/models/inkling uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider fireworks --model accounts/fireworks/models/inkling --setting standard uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider fireworks --model accounts/fireworks/models/inkling --setting vision @@ -279,24 +279,13 @@ bash benchmarks/mmmu_pro/run_smoke.sh Links: [paper](https://arxiv.org/abs/2311.12022) · [dataset](https://huggingface.co/datasets/Idavidrein/gpqa) (config: `gpqa_diamond`) ```bash -# OpenAI -uv run -m benchmarks.gpqa.gpqa_openai -uv run -m benchmarks.gpqa.gpqa_openai --model gpt-5.4-mini - -# Gemini -uv run -m benchmarks.gpqa.gpqa_gemini -uv run -m benchmarks.gpqa.gpqa_gemini --model gemini-3.1-pro-preview - -# Any model via OpenRouter (Grok, Kimi, Anthropic, etc.) -uv run -m benchmarks.gpqa.gpqa_openrouter --model x-ai/grok-4.3 --thinking on -uv run -m benchmarks.gpqa.gpqa_openrouter --model moonshotai/kimi-k2.6 - -# Fireworks (Inkling) -uv run -m benchmarks.gpqa.gpqa_fireworks -uv run -m benchmarks.gpqa.gpqa_fireworks --thinking on --effort high - -# Evaluate only -uv run -m benchmarks.gpqa.gpqa_openai --evaluate-only +# GPQA runs through the unified CLI. Provider, model, and reasoning defaults come +# from the target (bench_core/targets.yaml); pick any target by name. +uv run python -m bench_core list-targets +uv run python -m bench_core run --target gpt-5.5 --benchmark gpqa +uv run python -m bench_core run --target gemini-3.7-flash --benchmark gpqa --reasoning high +uv run python -m bench_core run --target grok-4.3 --benchmark gpqa +uv run python -m bench_core run --target inkling --benchmark gpqa ``` --- diff --git a/bench_core/__main__.py b/bench_core/__main__.py new file mode 100644 index 0000000..08837ce --- /dev/null +++ b/bench_core/__main__.py @@ -0,0 +1,3 @@ +from bench_core.cli import main + +main() diff --git a/bench_core/cli.py b/bench_core/cli.py new file mode 100644 index 0000000..199a025 --- /dev/null +++ b/bench_core/cli.py @@ -0,0 +1,132 @@ +"""bench CLI: run a benchmark against a target, or list targets.""" + +from __future__ import annotations + +import argparse +import asyncio +import importlib + +from bench_core.config import ( + build_adapter, + load_all_targets, + load_target, + resolve_capabilities, +) +from bench_core.results import RunStore +from bench_core.runner import run_benchmark + +# benchmark short-name -> module exposing NAME/load_samples/build_request/parse/score +BENCHMARKS = { + "gpqa": "benchmarks.gpqa.bench", +} + + +def _import_benchmark(name: str): + if name not in BENCHMARKS: + raise SystemExit(f"unknown benchmark {name!r}; known: {sorted(BENCHMARKS)}") + return importlib.import_module(BENCHMARKS[name]) + + +def cmd_run(args) -> None: + target = load_target(args.target) + bench = _import_benchmark(args.benchmark) + mode = args.reasoning or bench.DEFAULTS.get("reasoning", "off") + caps = resolve_capabilities(target, benchmark=args.benchmark) + adapter = build_adapter(target) + if not adapter.resolve_key(): + raise SystemExit( + f"no API key for provider {target.provider!r} (looked for {adapter.key_spec})" + ) + client = adapter.build_client() + + print(f"Loading {args.benchmark} samples...") + samples = bench.load_samples(args.sample) + store = RunStore(bench.NAME, target.name) + d = bench.DEFAULTS + + result = asyncio.run( + run_benchmark( + adapter=adapter, + client=client, + caps=caps, + model_id=target.model_id, + samples=samples, + build_request=lambda s: bench.build_request(s, mode), + parse=bench.parse, + store=store, + id_key=bench.ID_KEY, + rate_limit=d.get("rate_limit", 25), + max_in_flight=d.get("max_in_flight", 8), + ) + ) + + metrics = bench.score(store.load_responses(), samples) + metrics.update( + { + "benchmark": bench.NAME, + "target": target.name, + "provider": target.provider, + "model_id": target.model_id, + "n": metrics.get("total"), + "reasoning": { + "mode": mode, + "style": caps.reasoning.style.value, + "true_off": caps.reasoning.true_off, + }, + "capability_hints": result.hints, + } + ) + store.write_metrics(metrics) + store.write_run( + { + "provider": target.provider, + "model_id": target.model_id, + "reasoning_mode": mode, + "n_completed": result.n_completed, + "n_failed": result.n_failed, + "sample_size": args.sample, + } + ) + + acc = metrics.get("accuracy") + print( + f"\n{bench.NAME} / {target.name}: " + + (f"accuracy={acc:.4f} " if acc is not None else "") + + f"({metrics.get('correct')}/{metrics.get('total')}) failed={result.n_failed}" + ) + print(f"written to {store.metrics_path}") + if result.hints: + print("capability hints (promote to targets.yaml):") + for h in result.hints: + print(" -", h) + + +def cmd_list_targets(args) -> None: + for name, t in sorted(load_all_targets().items()): + print(f"{name:22} {t.provider:11} {t.model_id}") + + +def main(argv=None) -> None: + p = argparse.ArgumentParser(prog="bench") + sub = p.add_subparsers(dest="cmd", required=True) + + r = sub.add_parser("run", help="run a benchmark against a target") + r.add_argument("--target", required=True, help="target name (see list-targets)") + r.add_argument("--benchmark", required=True, choices=sorted(BENCHMARKS)) + r.add_argument( + "--reasoning", default=None, help="off|low|medium|high (default: benchmark's)" + ) + r.add_argument( + "--sample", type=int, default=None, help="run only the first N samples" + ) + r.set_defaults(func=cmd_run) + + lt = sub.add_parser("list-targets", help="list configured targets") + lt.set_defaults(func=cmd_list_targets) + + args = p.parse_args(argv) + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/gpqa/bench.py b/benchmarks/gpqa/bench.py new file mode 100644 index 0000000..2a5f2fe --- /dev/null +++ b/benchmarks/gpqa/bench.py @@ -0,0 +1,119 @@ +"""GPQA Diamond: 198 expert MCQs; deterministic choice shuffle; letter-match accuracy.""" + +from __future__ import annotations + +import random +import re +from collections import defaultdict + +from bench_core.request import Message, ReasoningSpec, Request, TextPart + +NAME = "gpqa" +ID_KEY = "id" +DEFAULTS = {"reasoning": "off", "rate_limit": 10, "max_in_flight": 8} + +_DATASET_ID = "Idavidrein/gpqa" +_CONFIG = "gpqa_diamond" +_SPLIT = "train" # GPQA Diamond ships as a single 'train' split (198 rows) + +_PROMPT = ( + "The following is a multiple choice question (with answers). Respond with " + "only the single letter (A, B, C, or D) corresponding to the correct " + "answer. Do not show your work.\n\n" + "Question: {question}\n" + "A. {a}\nB. {b}\nC. {c}\nD. {d}\n\n" + "Answer:" +) + +_LETTER_RE = re.compile(r"\b([ABCD])\b") +_FIRST_LETTER_RE = re.compile(r"[ABCD]") + + +def _clean(s) -> str: + return str(s).strip() if s is not None else "" + + +def _build_sample(row: dict) -> dict: + # Shuffle choices deterministically by Record ID (avoids position bias and + # eval-to-eval drift); track which letter holds the correct answer. + record_id = str(row.get("Record ID") or row.get("Question", "")) + correct = _clean(row["Correct Answer"]) + incorrect = [ + _clean(row["Incorrect Answer 1"]), + _clean(row["Incorrect Answer 2"]), + _clean(row["Incorrect Answer 3"]), + ] + choices = [(correct, True)] + [(x, False) for x in incorrect] + random.Random(record_id).shuffle(choices) + letters = ["A", "B", "C", "D"] + correct_letter = letters[next(i for i, (_, c) in enumerate(choices) if c)] + return { + "id": record_id, + "question": _clean(row["Question"]), + "a": choices[0][0], + "b": choices[1][0], + "c": choices[2][0], + "d": choices[3][0], + "correct_letter": correct_letter, + "domain": _clean(row.get("High-level domain")), + } + + +def load_samples(sample_size: int | None = None) -> list[dict]: + from datasets import load_dataset + + ds = load_dataset(_DATASET_ID, _CONFIG, split=_SPLIT) + samples = [_build_sample(dict(row)) for row in ds] + return samples[:sample_size] if sample_size else samples + + +def build_request(sample: dict, mode: str) -> Request: + prompt = _PROMPT.format( + question=sample["question"], + a=sample["a"], + b=sample["b"], + c=sample["c"], + d=sample["d"], + ) + return Request( + messages=[Message("user", [TextPart(prompt)])], + reasoning=ReasoningSpec(mode), + temperature=0.0, + ) + + +def parse(response, sample) -> str | None: + text = (response.text or "").strip() + if not text: + return None + if len(text) == 1 and text.upper() in "ABCD": + return text.upper() + up = text.upper() + m = _LETTER_RE.search(up) + if m: + return m.group(1) + m = _FIRST_LETTER_RE.search(up) + return m.group(0) if m else None + + +def score(records: list[dict], samples: list[dict]) -> dict: + by_id = {s["id"]: s for s in samples} + rows = [(r.get("prediction"), by_id[r["id"]]) for r in records if r["id"] in by_id] + total = len(rows) + correct = sum(1 for pred, s in rows if pred == s["correct_letter"]) + unparseable = sum(1 for pred, _ in rows if pred is None) + + by_domain: dict[str, list[bool]] = defaultdict(list) + for pred, s in rows: + by_domain[s["domain"] or "Unknown"].append(pred == s["correct_letter"]) + per_domain = { + d: {"n": len(v), "accuracy": sum(v) / len(v)} for d, v in by_domain.items() + } + + return { + "accuracy": correct / total if total else 0.0, + "correct": correct, + "total": total, + "unparseable": unparseable, + "per_domain": per_domain, + } diff --git a/benchmarks/gpqa/gpqa_fireworks.py b/benchmarks/gpqa/gpqa_fireworks.py deleted file mode 100644 index fbb4967..0000000 --- a/benchmarks/gpqa/gpqa_fireworks.py +++ /dev/null @@ -1,324 +0,0 @@ -""" -GPQA Diamond benchmark for Fireworks-hosted models (default: Inkling). - -Mirrors benchmarks.gpqa.gpqa_openai exactly — same dataset, same prompt, same -deterministic 4-choice shuffle, same parser, same metrics — but routes calls -through Fireworks' OpenAI-compatible endpoint. - -Inkling always thinks. `--thinking off` maps to reasoning_effort="none", which -is its floor rather than a true disable, so even "off" runs emit reasoning -tokens. `--thinking on` maps to reasoning_effort="high" (or --effort). - -Usage: - uv run -m benchmarks.gpqa.gpqa_fireworks - uv run -m benchmarks.gpqa.gpqa_fireworks --thinking on --effort high - uv run -m benchmarks.gpqa.gpqa_fireworks --model accounts/fireworks/models/inkling-small - uv run -m benchmarks.gpqa.gpqa_fireworks --limit 5 - uv run -m benchmarks.gpqa.gpqa_fireworks --evaluate-only - -Env: FIREWORKS_API_KEY. The dataset (Idavidrein/gpqa) is gated — accept its -terms on HuggingFace and log in (`hf auth login`) before running. -""" - -import re -import sys -import json -import time -import asyncio -import argparse -import traceback -from pathlib import Path - -from datasets import load_dataset -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -# Reuse all the dataset/parsing/metrics plumbing from the OpenAI runner. -from benchmarks.gpqa.gpqa_openai import ( # noqa: E402 - DATASET_ID, - CONFIG, - SPLIT, - PROMPT_TEMPLATE, - JsonlWriter, - load_completed_ids, - load_records, - parse_letter, - build_sample, - compute_metrics, -) -from src.commons_fireworks import INKLING, fireworks_client # noqa: E402 - -RESULTS_DIR = PROJECT_ROOT / "results" - -DEFAULT_MODEL = INKLING -DEFAULT_THINKING = "off" # 'on' or 'off' ('off' = reasoning_effort="none" floor) -DEFAULT_EFFORT: str | None = None # None | low | medium | high | xhigh | max -TEMPERATURE = 0.0 -CONCURRENCY = 10 -MAX_RETRIES = 5 -RETRY_BACKOFF_CAP_S = 30.0 - -MODEL = DEFAULT_MODEL -THINKING = DEFAULT_THINKING -EFFORT: str | None = DEFAULT_EFFORT - - -def model_slug(model: str) -> str: - """accounts/fireworks/models/inkling-small -> inklingsmall""" - return re.sub(r"[^a-z0-9]", "", model.rsplit("/", 1)[-1].lower()) - - -def invoke_fireworks(messages: list[dict]): - effort = "none" if THINKING == "off" else (EFFORT or "high") - return fireworks_client.chat.completions.create( - model=MODEL, - messages=messages, - temperature=TEMPERATURE, - reasoning_effort=effort, - ) - - -def _reasoning_tokens(response) -> int | None: - u = getattr(response, "usage", None) - if u is None: - return None - details = getattr(u, "completion_tokens_details", None) - val = getattr(details, "reasoning_tokens", None) if details else None - if val is None: - # Fireworks reports it at the top level of `usage` for Inkling. - val = getattr(u, "reasoning_tokens", None) - try: - return int(val) if val is not None else None - except (TypeError, ValueError): - return None - - -async def process_sample( - sample: dict, semaphore: asyncio.Semaphore, writer: JsonlWriter, progress: dict -) -> dict | None: - prompt = PROMPT_TEMPLATE.format( - question=sample["question"], - a=sample["a"], - b=sample["b"], - c=sample["c"], - d=sample["d"], - ) - messages = [{"role": "user", "content": prompt}] - last_error: str | None = None - # Recorded per sample: stdout can be truncated by whatever captures it, and - # "why did half the run retry" is only answerable from the checkpoint. - retry_errors: list[str] = [] - - for attempt in range(1, MAX_RETRIES + 1): - start = time.perf_counter() - try: - async with semaphore: - start = time.perf_counter() - response = await asyncio.to_thread(invoke_fireworks, messages) - latency_ms = int((time.perf_counter() - start) * 1000) - content = (response.choices[0].message.content or "").strip() - request_id = getattr(response, "id", None) - if not content: - last_error = "empty response content" - raise RuntimeError(last_error) - - predicted = parse_letter(content) - correct = predicted == sample["correct_letter"] - reasoning_tokens = _reasoning_tokens(response) - - record = { - "id": sample["id"], - "domain": sample["domain"], - "subdomain": sample["subdomain"], - "correct_letter": sample["correct_letter"], - "prediction": predicted, - "correct": correct, - "response": content, - "request_id": request_id, - "latency_ms": latency_ms, - "attempts": attempt, - "reasoning_tokens": reasoning_tokens, - "retry_errors": retry_errors, - } - await writer.append(record) - - progress["done"] += 1 - if correct: - progress["correct"] += 1 - mark = "OK" if correct else "X " - tqdm.write( - f"[{progress['done']}/{progress['total']}] {mark} " - f"id={sample['id']} domain={sample['domain']:10} " - f"gold={sample['correct_letter']} pred={predicted or '?'} " - f"latency={latency_ms}ms reasoning_tok={reasoning_tokens if reasoning_tokens is not None else '?'} " - f"attempt={attempt}" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - retry_errors.append(last_error[:200]) - tqdm.write( - f"[error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(min(2 ** (attempt - 1), RETRY_BACKOFF_CAP_S)) - - progress["failed"] += 1 - tqdm.write(f"[FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") - return None - - -def print_summary(metrics: dict): - print(f"\n{'=' * 60}") - print(f"GPQA Diamond — {DATASET_ID}/{CONFIG} ({MODEL}, thinking={THINKING})") - print(f"{'=' * 60}") - print( - f"Accuracy : {metrics['accuracy']:.4f} ({metrics['correct']}/{metrics['total']})" - ) - print(f"Unparseable: {metrics['unparseable']}") - print("\nPer high-level domain:") - for d in sorted(metrics["per_domain"]): - v = metrics["per_domain"][d] - print(f" {d:12} n={v['n']:>3} acc={v['accuracy']:.4f}") - if metrics.get("latency"): - lat = metrics["latency"] - print( - f"\nLatency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " - f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms max={lat['max_ms']}ms" - ) - - -async def run_predictions(pred_path: Path, limit: int | None): - print(f"Loading {DATASET_ID}/{CONFIG} (split={SPLIT})...") - ds = load_dataset(DATASET_ID, CONFIG, split=SPLIT) - print(f"Loaded {len(ds)} rows") - samples = [build_sample(dict(row)) for row in ds] - - done_ids = load_completed_ids(pred_path) - pending = [s for s in samples if s["id"] not in done_ids] - if limit is not None: - pending = pending[:limit] - print(f"--limit applied: will run at most {limit} sample(s)") - print( - f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})" - ) - if not pending: - return - - writer = JsonlWriter(pred_path) - semaphore = asyncio.Semaphore(CONCURRENCY) - progress = {"total": len(pending), "done": 0, "correct": 0, "failed": 0} - - tasks = [process_sample(s, semaphore, writer, progress) for s in pending] - try: - await tqdm_asyncio.gather(*tasks, desc=f"GPQA Diamond / {MODEL}") - except Exception: - traceback.print_exc() - acc = progress["correct"] / progress["done"] if progress["done"] else 0.0 - print( - f"\nRun finished: {progress['done']}/{progress['total']} answered, " - f"{progress['correct']} correct (acc={acc:.4f}), {progress['failed']} failed." - ) - - -def run_evaluation(pred_path: Path, metrics_path: Path): - if not pred_path.exists(): - print(f"No predictions found at {pred_path}") - sys.exit(1) - results = load_records(pred_path) - if not results: - print(f"No records in {pred_path}") - sys.exit(1) - for r in results: - if r.get("prediction") is None and r.get("response"): - r["prediction"] = parse_letter(r["response"]) - if r.get("correct") is None and r.get("prediction") is not None: - r["correct"] = r["prediction"] == r.get("correct_letter") - - metrics = compute_metrics(results) - print_summary(metrics) - reasoning_toks = [ - r["reasoning_tokens"] for r in results if r.get("reasoning_tokens") is not None - ] - output = { - **metrics, - "dataset": DATASET_ID, - "config": CONFIG, - "split": SPLIT, - "model": MODEL, - "thinking": THINKING, - "effort": "none" if THINKING == "off" else (EFFORT or "high"), - "temperature": TEMPERATURE, - "concurrency": CONCURRENCY, - "provider": "fireworks", - "mean_reasoning_tokens": ( - sum(reasoning_toks) / len(reasoning_toks) if reasoning_toks else None - ), - } - metrics_path.parent.mkdir(parents=True, exist_ok=True) - with open(metrics_path, "w") as f: - json.dump(output, f, indent=2, ensure_ascii=False) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - global MODEL, THINKING, EFFORT - parser = argparse.ArgumentParser( - description="GPQA Diamond benchmark for Fireworks-hosted models" - ) - parser.add_argument( - "--model", - default=DEFAULT_MODEL, - help="Fireworks model id (accounts/fireworks/...)", - ) - parser.add_argument( - "--thinking", - default=DEFAULT_THINKING, - choices=["on", "off"], - help="off = reasoning_effort 'none' (Inkling's floor); on = 'high' or --effort", - ) - parser.add_argument( - "--effort", - default=DEFAULT_EFFORT, - choices=["low", "medium", "high", "xhigh", "max"], - help="Explicit reasoning_effort. Only applied when thinking=on.", - ) - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument( - "--limit", - type=int, - default=None, - help="Only run the first N unanswered samples", - ) - args = parser.parse_args() - - MODEL = args.model - THINKING = args.thinking - EFFORT = args.effort - - effort_suffix = f"_effort{EFFORT}" if (THINKING == "on" and EFFORT) else "" - tag = ( - f"fireworks_{model_slug(MODEL)}_thinking{THINKING}{effort_suffix}_gpqa_diamond" - ) - pred_path = RESULTS_DIR / f"{tag}_responses.jsonl" - metrics_path = RESULTS_DIR / f"{tag}_metrics.json" - - if args.evaluate_only: - run_evaluation(pred_path, metrics_path) - elif args.predict_only: - asyncio.run(run_predictions(pred_path, limit=args.limit)) - else: - asyncio.run(run_predictions(pred_path, limit=args.limit)) - run_evaluation(pred_path, metrics_path) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/gpqa/gpqa_gemini.py b/benchmarks/gpqa/gpqa_gemini.py deleted file mode 100644 index 9eb5c70..0000000 --- a/benchmarks/gpqa/gpqa_gemini.py +++ /dev/null @@ -1,277 +0,0 @@ -""" -GPQA Diamond benchmark for Gemini. - -198 graduate-level multiple-choice questions in physics, chemistry, and biology. -Mirrors `gpqa_openai.py` exactly — same prompt, same deterministic shuffle of -choices, same scoring — only the inference call differs. - -Default model: gemini-3.1-pro-preview. Thinking is left ON at the model's -default level (the Pro family is reasoning-first; we don't pass a -thinking_config). Temperature pinned to 0.0. - -Usage: - uv run -m benchmarks.gpqa.gpqa_gemini - uv run -m benchmarks.gpqa.gpqa_gemini --model gemini-3.1-pro-preview - uv run -m benchmarks.gpqa.gpqa_gemini --thinking-level minimal - uv run -m benchmarks.gpqa.gpqa_gemini --limit 5 - uv run -m benchmarks.gpqa.gpqa_gemini --evaluate-only - -Env: GEMINI_KEY must be set in .env. -""" - -import os -import re -import sys -import json -import time -import asyncio -import argparse -import traceback -from pathlib import Path -from collections import defaultdict - -from datasets import load_dataset -from dotenv import load_dotenv -from google import genai -from google.genai import types -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -# Reuse parsing/sample-builder/scoring from the OpenAI variant — guarantees -# identical prompts and answer-letter parsing across providers. -from benchmarks.gpqa.gpqa_openai import ( # noqa: E402 - DATASET_ID, - CONFIG, - SPLIT, - PROMPT_TEMPLATE, - JsonlWriter, - build_sample, - compute_metrics, - load_completed_ids, - load_records, - parse_letter, -) - -load_dotenv() - -RESULTS_DIR = PROJECT_ROOT / "results" -DEFAULT_MODEL = "gemini-3.1-pro-preview" -DEFAULT_THINKING_LEVEL: str | None = None # None => model default (Pro: thinking on) -TEMPERATURE = 0.0 -CONCURRENCY = 30 # Flash has much higher headroom than Pro. -MAX_RETRIES = 6 -RETRY_BACKOFF_CAP_S = 30.0 - -MODEL = DEFAULT_MODEL -THINKING_LEVEL: str | None = DEFAULT_THINKING_LEVEL - -GEMINI_KEY = ( - os.getenv("GEMINI_KEY") or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY") -) -if not GEMINI_KEY: - raise RuntimeError( - "GEMINI_KEY is not set. Add it to .env " - "(get one from https://aistudio.google.com/app/apikey)." - ) - -gemini_client = genai.Client(api_key=GEMINI_KEY) - - -def model_slug(model: str) -> str: - return re.sub(r"[^a-z0-9]", "", model.lower()) - - -def invoke_gemini(prompt: str): - config_kwargs = {"temperature": TEMPERATURE} - if THINKING_LEVEL is not None: - config_kwargs["thinking_config"] = types.ThinkingConfig(thinking_level=THINKING_LEVEL) - config = types.GenerateContentConfig(**config_kwargs) - return gemini_client.models.generate_content( - model=MODEL, - contents=[types.Part.from_text(text=prompt)], - config=config, - ) - - -async def process_sample(sample: dict, semaphore: asyncio.Semaphore, - writer: JsonlWriter, progress: dict) -> dict | None: - prompt = PROMPT_TEMPLATE.format( - question=sample["question"], - a=sample["a"], b=sample["b"], c=sample["c"], d=sample["d"], - ) - last_error: str | None = None - - for attempt in range(1, MAX_RETRIES + 1): - start = time.perf_counter() - try: - async with semaphore: - start = time.perf_counter() - response = await asyncio.to_thread(invoke_gemini, prompt) - latency_ms = int((time.perf_counter() - start) * 1000) - content = (response.text or "").strip() - request_id = getattr(response, "response_id", None) - if not content: - last_error = "empty response content" - raise RuntimeError(last_error) - - predicted = parse_letter(content) - correct = predicted == sample["correct_letter"] - - record = { - "id": sample["id"], - "domain": sample["domain"], - "subdomain": sample["subdomain"], - "correct_letter": sample["correct_letter"], - "prediction": predicted, - "correct": correct, - "response": content, - "request_id": request_id, - "latency_ms": latency_ms, - "attempts": attempt, - } - await writer.append(record) - - progress["done"] += 1 - if correct: - progress["correct"] += 1 - mark = "OK" if correct else "X " - tqdm.write( - f"[{progress['done']}/{progress['total']}] {mark} " - f"id={sample['id']} domain={sample['domain']:10} " - f"gold={sample['correct_letter']} pred={predicted or '?'} " - f"latency={latency_ms}ms attempt={attempt}" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - tqdm.write( - f"[error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(min(2 ** (attempt - 1), RETRY_BACKOFF_CAP_S)) - - progress["failed"] += 1 - tqdm.write(f"[FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") - return None - - -def print_summary(metrics: dict): - print(f"\n{'=' * 60}") - print( - f"GPQA Diamond — {DATASET_ID}/{CONFIG} ({MODEL}, " - f"thinking={'default' if THINKING_LEVEL is None else THINKING_LEVEL}, " - f"temp={TEMPERATURE})" - ) - print(f"{'=' * 60}") - print(f"Accuracy : {metrics['accuracy']:.4f} ({metrics['correct']}/{metrics['total']})") - print(f"Unparseable: {metrics['unparseable']}") - print("\nPer high-level domain:") - for d in sorted(metrics["per_domain"]): - v = metrics["per_domain"][d] - print(f" {d:12} n={v['n']:>3} acc={v['accuracy']:.4f}") - if metrics.get("latency"): - lat = metrics["latency"] - print(f"\nLatency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " - f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms max={lat['max_ms']}ms") - - -async def run_predictions(pred_path: Path, limit: int | None): - print(f"Loading {DATASET_ID}/{CONFIG} (split={SPLIT})...") - ds = load_dataset(DATASET_ID, CONFIG, split=SPLIT) - print(f"Loaded {len(ds)} rows") - samples = [build_sample(dict(row)) for row in ds] - - done_ids = load_completed_ids(pred_path) - pending = [s for s in samples if s["id"] not in done_ids] - if limit is not None: - pending = pending[:limit] - print(f"--limit applied: will run at most {limit} sample(s)") - print(f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})") - if not pending: - return - - writer = JsonlWriter(pred_path) - semaphore = asyncio.Semaphore(CONCURRENCY) - progress = {"total": len(pending), "done": 0, "correct": 0, "failed": 0} - - tasks = [process_sample(s, semaphore, writer, progress) for s in pending] - try: - await tqdm_asyncio.gather(*tasks, desc=f"GPQA Diamond / {MODEL}") - except Exception: - traceback.print_exc() - acc = progress["correct"] / progress["done"] if progress["done"] else 0.0 - print(f"\nRun finished: {progress['done']}/{progress['total']} answered, " - f"{progress['correct']} correct (acc={acc:.4f}), {progress['failed']} failed.") - - -def run_evaluation(pred_path: Path, metrics_path: Path): - if not pred_path.exists(): - print(f"No predictions found at {pred_path}") - sys.exit(1) - results = load_records(pred_path) - if not results: - print(f"No records in {pred_path}") - sys.exit(1) - for r in results: - if r.get("prediction") is None and r.get("response"): - r["prediction"] = parse_letter(r["response"]) - if r.get("correct") is None and r.get("prediction") is not None: - r["correct"] = r["prediction"] == r.get("correct_letter") - - metrics = compute_metrics(results) - print_summary(metrics) - output = { - **metrics, - "dataset": DATASET_ID, - "config": CONFIG, - "split": SPLIT, - "model": MODEL, - "thinking_level": THINKING_LEVEL, - "temperature": TEMPERATURE, - "concurrency": CONCURRENCY, - } - metrics_path.parent.mkdir(parents=True, exist_ok=True) - with open(metrics_path, "w") as f: - json.dump(output, f, indent=2, ensure_ascii=False) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - global MODEL, THINKING_LEVEL - parser = argparse.ArgumentParser(description="GPQA Diamond benchmark for Gemini") - parser.add_argument("--model", default=DEFAULT_MODEL, - help="Gemini model id (e.g. gemini-3.1-pro-preview)") - parser.add_argument("--thinking-level", default=DEFAULT_THINKING_LEVEL, - help="thinking_level ('minimal'|'low'|'high'); omit for model default") - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument("--limit", type=int, default=None, - help="Only run the first N unanswered samples") - args = parser.parse_args() - - MODEL = args.model - THINKING_LEVEL = args.thinking_level - - thinking_slug = THINKING_LEVEL or "default" - tag = f"{model_slug(MODEL)}_thinking{thinking_slug}_gpqa_diamond" - pred_path = RESULTS_DIR / f"{tag}_responses.jsonl" - metrics_path = RESULTS_DIR / f"{tag}_metrics.json" - - if args.evaluate_only: - run_evaluation(pred_path, metrics_path) - elif args.predict_only: - asyncio.run(run_predictions(pred_path, limit=args.limit)) - else: - asyncio.run(run_predictions(pred_path, limit=args.limit)) - run_evaluation(pred_path, metrics_path) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/gpqa/gpqa_openai.py b/benchmarks/gpqa/gpqa_openai.py deleted file mode 100644 index fe62940..0000000 --- a/benchmarks/gpqa/gpqa_openai.py +++ /dev/null @@ -1,467 +0,0 @@ -""" -GPQA Diamond benchmark for OpenAI GPT-5.x. - -198 graduate-level multiple-choice questions in physics, chemistry, and biology -written and validated by domain experts. The "Diamond" subset is the hardest -slice of GPQA — questions where both expert validators answered correctly and -the majority of non-experts answered incorrectly. - -Dataset: https://huggingface.co/datasets/Idavidrein/gpqa (config: gpqa_diamond) -Paper: https://arxiv.org/abs/2311.12022 - -Methodology: - - For each question we deterministically shuffle the 4 answer choices using - Record ID as the seed. This avoids both position bias (always-A) and - pure-random eval-to-eval drift. - - Single-shot, pass@1, no chain-of-thought prompted. The model is asked to - output a single letter A/B/C/D. - - Metric: exact-match accuracy on the predicted letter, reported overall and - per high-level domain. - -Default model: gpt-5.5. Run gpt-5.4-mini with `--model gpt-5.4-mini`. -Reasoning defaults to fully off (`reasoning_effort="none"`); temperature 0.0. - -Usage: - uv run -m benchmarks.gpqa.gpqa_openai - uv run -m benchmarks.gpqa.gpqa_openai --model gpt-5.4-mini - uv run -m benchmarks.gpqa.gpqa_openai --limit 5 - uv run -m benchmarks.gpqa.gpqa_openai --evaluate-only - -Env: OPENAI_API_KEY must be set. -""" - -import os -import re -import sys -import json -import time -import random -import asyncio -import argparse -import traceback -from pathlib import Path -from collections import defaultdict - -from datasets import load_dataset -from dotenv import load_dotenv -from openai import OpenAI -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -load_dotenv() - -RESULTS_DIR = PROJECT_ROOT / "results" -DATASET_ID = "Idavidrein/gpqa" -CONFIG = "gpqa_diamond" -SPLIT = "train" # GPQA Diamond ships as a single 'train' split with 198 rows. - -DEFAULT_MODEL = "gpt-5.5" -DEFAULT_REASONING_EFFORT = "none" -TEMPERATURE = 0.0 -CONCURRENCY = 10 -MAX_RETRIES = 5 -RETRY_BACKOFF_CAP_S = 30.0 - -MODEL = DEFAULT_MODEL -REASONING_EFFORT = DEFAULT_REASONING_EFFORT - -PROMPT_TEMPLATE = ( - "The following is a multiple choice question (with answers). Respond with " - "only the single letter (A, B, C, or D) corresponding to the correct " - "answer. Do not show your work.\n\n" - "Question: {question}\n" - "A. {a}\n" - "B. {b}\n" - "C. {c}\n" - "D. {d}\n\n" - "Answer:" -) - -OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") - -# Bypass the .env's interfaze base_url override. Built only when the key is -# present — the openrouter/fireworks runners import this module's dataset and -# scoring helpers, so a hard raise here would block them; invoke_openai raises -# instead. -openai_client = ( - OpenAI(base_url="https://api.openai.com/v1", api_key=OPENAI_API_KEY) - if OPENAI_API_KEY - else None -) - - -def model_slug(model: str) -> str: - return re.sub(r"[^a-z0-9]", "", model.lower()) - - -def invoke_openai(messages: list[dict]): - if openai_client is None: - raise RuntimeError( - "OPENAI_API_KEY is not set. Add it to .env " - "(get one from https://platform.openai.com/api-keys)." - ) - # GPT-5.x rejects temperature!=default when reasoning is engaged - # ("Unsupported value: 'temperature' does not support 0.0 with this model. - # Only the default (1) value is supported."). With reasoning off ('none'), - # temperature=0 is accepted and gives us deterministic decoding. - kwargs = { - "model": MODEL, - "messages": messages, - "reasoning_effort": REASONING_EFFORT, - } - if REASONING_EFFORT == "none": - kwargs["temperature"] = TEMPERATURE - return openai_client.chat.completions.create(**kwargs) - - -class JsonlWriter: - def __init__(self, path: Path): - self.path = path - self.path.parent.mkdir(parents=True, exist_ok=True) - self._lock = asyncio.Lock() - - async def append(self, record: dict): - line = json.dumps(record, ensure_ascii=False) - async with self._lock: - with open(self.path, "a", encoding="utf-8") as f: - f.write(line + "\n") - f.flush() - os.fsync(f.fileno()) - - -def load_completed_ids(path: Path) -> set[str]: - if not path.exists(): - return set() - done: set[str] = set() - with open(path, encoding="utf-8") as f: - for line_no, line in enumerate(f, 1): - line = line.strip() - if not line: - continue - try: - rec = json.loads(line) - except json.JSONDecodeError: - tqdm.write(f"[resume] skipping malformed line {line_no} in {path}") - continue - if rec.get("response") is not None: - done.add(str(rec["id"])) - return done - - -def load_records(path: Path) -> list[dict]: - by_id: dict[str, dict] = {} - if not path.exists(): - return [] - with open(path, encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - rec = json.loads(line) - by_id[str(rec["id"])] = rec - except json.JSONDecodeError: - continue - return list(by_id.values()) - - -_LETTER_RE = re.compile(r"\b([ABCD])\b") -_FIRST_LETTER_RE = re.compile(r"[ABCD]") - - -def parse_letter(text: str) -> str | None: - if not text: - return None - s = text.strip() - if len(s) == 1 and s.upper() in "ABCD": - return s.upper() - m = _LETTER_RE.search(s.upper()) - if m: - return m.group(1) - m = _FIRST_LETTER_RE.search(s.upper()) - if m: - return m.group(0) - return None - - -def _clean(s) -> str: - return str(s).strip() if s is not None else "" - - -def build_sample(row: dict) -> dict: - """Shuffle the 4 choices deterministically using Record ID as seed; track - which letter ended up holding the correct answer.""" - record_id = str(row.get("Record ID") or row.get("Question", "")) - correct = _clean(row["Correct Answer"]) - incorrect = [ - _clean(row["Incorrect Answer 1"]), - _clean(row["Incorrect Answer 2"]), - _clean(row["Incorrect Answer 3"]), - ] - choices = [(correct, True)] + [(x, False) for x in incorrect] - rng = random.Random(record_id) - rng.shuffle(choices) - letters = ["A", "B", "C", "D"] - correct_letter = letters[next(i for i, (_, is_c) in enumerate(choices) if is_c)] - return { - "id": record_id, - "question": _clean(row["Question"]), - "a": choices[0][0], - "b": choices[1][0], - "c": choices[2][0], - "d": choices[3][0], - "correct_letter": correct_letter, - "correct_answer_text": correct, - "domain": _clean(row.get("High-level domain")), - "subdomain": _clean(row.get("Subdomain")), - } - - -async def process_sample( - sample: dict, semaphore: asyncio.Semaphore, writer: JsonlWriter, progress: dict -) -> dict | None: - prompt = PROMPT_TEMPLATE.format( - question=sample["question"], - a=sample["a"], - b=sample["b"], - c=sample["c"], - d=sample["d"], - ) - messages = [{"role": "user", "content": prompt}] - last_error: str | None = None - - for attempt in range(1, MAX_RETRIES + 1): - start = time.perf_counter() - try: - async with semaphore: - start = time.perf_counter() - response = await asyncio.to_thread(invoke_openai, messages) - latency_ms = int((time.perf_counter() - start) * 1000) - content = (response.choices[0].message.content or "").strip() - request_id = getattr(response, "id", None) - if not content: - last_error = "empty response content" - raise RuntimeError(last_error) - - predicted = parse_letter(content) - correct = predicted == sample["correct_letter"] - - record = { - "id": sample["id"], - "domain": sample["domain"], - "subdomain": sample["subdomain"], - "correct_letter": sample["correct_letter"], - "prediction": predicted, - "correct": correct, - "response": content, - "request_id": request_id, - "latency_ms": latency_ms, - "attempts": attempt, - } - await writer.append(record) - - progress["done"] += 1 - if correct: - progress["correct"] += 1 - mark = "OK" if correct else "X " - tqdm.write( - f"[{progress['done']}/{progress['total']}] {mark} " - f"id={sample['id']} domain={sample['domain']:10} " - f"gold={sample['correct_letter']} pred={predicted or '?'} " - f"latency={latency_ms}ms attempt={attempt}" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - tqdm.write( - f"[error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(min(2 ** (attempt - 1), RETRY_BACKOFF_CAP_S)) - - progress["failed"] += 1 - tqdm.write(f"[FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") - return None - - -def compute_metrics(results: list[dict]) -> dict: - if not results: - return {} - total = len(results) - n_correct = sum(1 for r in results if r.get("correct")) - n_unparseable = sum(1 for r in results if r.get("prediction") is None) - - by_domain: dict[str, list[dict]] = defaultdict(list) - for r in results: - by_domain[r.get("domain") or "Unknown"].append(r) - per_domain = { - d: { - "n": len(rows), - "accuracy": sum(1 for r in rows if r.get("correct")) / len(rows), - } - for d, rows in by_domain.items() - } - - latencies = [ - r["latency_ms"] for r in results if isinstance(r.get("latency_ms"), int) - ] - latency_stats = {} - if latencies: - lats = sorted(latencies) - n = len(lats) - latency_stats = { - "count": n, - "mean_ms": sum(lats) / n, - "p50_ms": lats[n // 2], - "p90_ms": lats[min(n - 1, int(n * 0.9))], - "p99_ms": lats[min(n - 1, int(n * 0.99))], - "max_ms": lats[-1], - } - - return { - "accuracy": n_correct / total, - "correct": n_correct, - "total": total, - "unparseable": n_unparseable, - "per_domain": per_domain, - "latency": latency_stats, - } - - -def print_summary(metrics: dict): - print(f"\n{'=' * 60}") - print( - f"GPQA Diamond — {DATASET_ID}/{CONFIG} ({MODEL}, reasoning={REASONING_EFFORT})" - ) - print(f"{'=' * 60}") - print( - f"Accuracy : {metrics['accuracy']:.4f} ({metrics['correct']}/{metrics['total']})" - ) - print(f"Unparseable: {metrics['unparseable']}") - print("\nPer high-level domain:") - for d in sorted(metrics["per_domain"]): - v = metrics["per_domain"][d] - print(f" {d:12} n={v['n']:>3} acc={v['accuracy']:.4f}") - if metrics.get("latency"): - lat = metrics["latency"] - print( - f"\nLatency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " - f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms max={lat['max_ms']}ms" - ) - - -async def run_predictions(pred_path: Path, limit: int | None): - print(f"Loading {DATASET_ID}/{CONFIG} (split={SPLIT})...") - ds = load_dataset(DATASET_ID, CONFIG, split=SPLIT) - print(f"Loaded {len(ds)} rows") - samples = [build_sample(dict(row)) for row in ds] - - done_ids = load_completed_ids(pred_path) - pending = [s for s in samples if s["id"] not in done_ids] - if limit is not None: - pending = pending[:limit] - print(f"--limit applied: will run at most {limit} sample(s)") - print( - f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})" - ) - if not pending: - return - - writer = JsonlWriter(pred_path) - semaphore = asyncio.Semaphore(CONCURRENCY) - progress = {"total": len(pending), "done": 0, "correct": 0, "failed": 0} - - tasks = [process_sample(s, semaphore, writer, progress) for s in pending] - try: - await tqdm_asyncio.gather(*tasks, desc=f"GPQA Diamond / {MODEL}") - except Exception: - traceback.print_exc() - acc = progress["correct"] / progress["done"] if progress["done"] else 0.0 - print( - f"\nRun finished: {progress['done']}/{progress['total']} answered, " - f"{progress['correct']} correct (acc={acc:.4f}), {progress['failed']} failed." - ) - - -def run_evaluation(pred_path: Path, metrics_path: Path): - if not pred_path.exists(): - print(f"No predictions found at {pred_path}") - sys.exit(1) - results = load_records(pred_path) - if not results: - print(f"No records in {pred_path}") - sys.exit(1) - for r in results: - if r.get("prediction") is None and r.get("response"): - r["prediction"] = parse_letter(r["response"]) - if r.get("correct") is None and r.get("prediction") is not None: - r["correct"] = r["prediction"] == r.get("correct_letter") - - metrics = compute_metrics(results) - print_summary(metrics) - output = { - **metrics, - "dataset": DATASET_ID, - "config": CONFIG, - "split": SPLIT, - "model": MODEL, - "reasoning_effort": REASONING_EFFORT, - "temperature": TEMPERATURE, - "concurrency": CONCURRENCY, - } - metrics_path.parent.mkdir(parents=True, exist_ok=True) - with open(metrics_path, "w") as f: - json.dump(output, f, indent=2, ensure_ascii=False) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - global MODEL, REASONING_EFFORT - parser = argparse.ArgumentParser( - description="GPQA Diamond benchmark for OpenAI GPT-5.x" - ) - parser.add_argument( - "--model", - default=DEFAULT_MODEL, - help="OpenAI model id (e.g. gpt-5.5, gpt-5.4-mini)", - ) - parser.add_argument( - "--reasoning-effort", - default=DEFAULT_REASONING_EFFORT, - help="reasoning_effort param ('none', 'minimal', 'low', 'medium', 'high', etc.)", - ) - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument( - "--limit", - type=int, - default=None, - help="Only run the first N unanswered samples", - ) - args = parser.parse_args() - - MODEL = args.model - REASONING_EFFORT = args.reasoning_effort - - tag = f"{model_slug(MODEL)}_reasoning{REASONING_EFFORT}_gpqa_diamond" - pred_path = RESULTS_DIR / f"{tag}_responses.jsonl" - metrics_path = RESULTS_DIR / f"{tag}_metrics.json" - - if args.evaluate_only: - run_evaluation(pred_path, metrics_path) - elif args.predict_only: - asyncio.run(run_predictions(pred_path, limit=args.limit)) - else: - asyncio.run(run_predictions(pred_path, limit=args.limit)) - run_evaluation(pred_path, metrics_path) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/gpqa/gpqa_openrouter.py b/benchmarks/gpqa/gpqa_openrouter.py deleted file mode 100644 index 31ed0ed..0000000 --- a/benchmarks/gpqa/gpqa_openrouter.py +++ /dev/null @@ -1,276 +0,0 @@ -""" -GPQA Diamond benchmark via OpenRouter (default: x-ai/grok-4.3 with thinking on). - -Mirrors benchmarks.gpqa.gpqa_openai exactly — same dataset, same prompt, same -deterministic 4-choice shuffle, same parser, same metrics — but routes calls -through OpenRouter's OpenAI-compatible endpoint so we can target any model -they host (xAI Grok, Anthropic, etc.). - -Thinking is toggled via OpenRouter's `reasoning` body field, passed through -the OpenAI SDK's extra_body. For Grok 4.x reasoning is on by default; we set -it explicitly so the run is reproducible regardless of provider defaults. - -Usage: - uv run -m benchmarks.gpqa.gpqa_openrouter - uv run -m benchmarks.gpqa.gpqa_openrouter --model x-ai/grok-4.3 --thinking on - uv run -m benchmarks.gpqa.gpqa_openrouter --limit 5 - uv run -m benchmarks.gpqa.gpqa_openrouter --evaluate-only - -Env: OPENROUTER_API_KEY must be set. -""" - -import os -import re -import sys -import json -import time -import asyncio -import argparse -import traceback -from pathlib import Path - -from datasets import load_dataset -from dotenv import load_dotenv -from openai import OpenAI -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -# Reuse all the dataset/parsing/metrics plumbing from the OpenAI runner. -from benchmarks.gpqa.gpqa_openai import ( # noqa: E402 - DATASET_ID, CONFIG, SPLIT, PROMPT_TEMPLATE, - JsonlWriter, load_completed_ids, load_records, - parse_letter, build_sample, compute_metrics, -) - -load_dotenv() - -RESULTS_DIR = PROJECT_ROOT / "results" - -DEFAULT_MODEL = "x-ai/grok-4.3" -DEFAULT_THINKING = "on" # 'on' or 'off' -DEFAULT_EFFORT: str | None = None # None | 'low' | 'medium' | 'high' -TEMPERATURE = 0.0 -CONCURRENCY = 10 -MAX_RETRIES = 5 -RETRY_BACKOFF_CAP_S = 30.0 - -MODEL = DEFAULT_MODEL -THINKING = DEFAULT_THINKING -EFFORT: str | None = DEFAULT_EFFORT - -OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY") or os.getenv( - "openrouter_api_key" -) -if not OPENROUTER_API_KEY: - raise RuntimeError( - "OPENROUTER_API_KEY is not set. Add it to .env " - "(get one from https://openrouter.ai/keys)." - ) - -openrouter_client = OpenAI( - base_url="https://openrouter.ai/api/v1", - api_key=OPENROUTER_API_KEY, -) - - -def model_slug(model: str) -> str: - return re.sub(r"[^a-z0-9]", "", model.lower()) - - -def invoke_openrouter(messages: list[dict]): - reasoning: dict = {"enabled": THINKING == "on"} - if THINKING == "on" and EFFORT: - reasoning["effort"] = EFFORT - extra_body = {"reasoning": reasoning} - return openrouter_client.chat.completions.create( - model=MODEL, - messages=messages, - temperature=TEMPERATURE, - extra_body=extra_body, - ) - - -async def process_sample(sample: dict, semaphore: asyncio.Semaphore, - writer: JsonlWriter, progress: dict) -> dict | None: - prompt = PROMPT_TEMPLATE.format( - question=sample["question"], - a=sample["a"], b=sample["b"], c=sample["c"], d=sample["d"], - ) - messages = [{"role": "user", "content": prompt}] - last_error: str | None = None - - for attempt in range(1, MAX_RETRIES + 1): - start = time.perf_counter() - try: - async with semaphore: - start = time.perf_counter() - response = await asyncio.to_thread(invoke_openrouter, messages) - latency_ms = int((time.perf_counter() - start) * 1000) - content = (response.choices[0].message.content or "").strip() - request_id = getattr(response, "id", None) - if not content: - last_error = "empty response content" - raise RuntimeError(last_error) - - predicted = parse_letter(content) - correct = predicted == sample["correct_letter"] - - record = { - "id": sample["id"], - "domain": sample["domain"], - "subdomain": sample["subdomain"], - "correct_letter": sample["correct_letter"], - "prediction": predicted, - "correct": correct, - "response": content, - "request_id": request_id, - "latency_ms": latency_ms, - "attempts": attempt, - } - await writer.append(record) - - progress["done"] += 1 - if correct: - progress["correct"] += 1 - mark = "OK" if correct else "X " - tqdm.write( - f"[{progress['done']}/{progress['total']}] {mark} " - f"id={sample['id']} domain={sample['domain']:10} " - f"gold={sample['correct_letter']} pred={predicted or '?'} " - f"latency={latency_ms}ms attempt={attempt}" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - tqdm.write( - f"[error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(min(2 ** (attempt - 1), RETRY_BACKOFF_CAP_S)) - - progress["failed"] += 1 - tqdm.write(f"[FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") - return None - - -def print_summary(metrics: dict): - print(f"\n{'=' * 60}") - print(f"GPQA Diamond — {DATASET_ID}/{CONFIG} ({MODEL}, thinking={THINKING})") - print(f"{'=' * 60}") - print(f"Accuracy : {metrics['accuracy']:.4f} ({metrics['correct']}/{metrics['total']})") - print(f"Unparseable: {metrics['unparseable']}") - print("\nPer high-level domain:") - for d in sorted(metrics["per_domain"]): - v = metrics["per_domain"][d] - print(f" {d:12} n={v['n']:>3} acc={v['accuracy']:.4f}") - if metrics.get("latency"): - lat = metrics["latency"] - print(f"\nLatency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " - f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms max={lat['max_ms']}ms") - - -async def run_predictions(pred_path: Path, limit: int | None): - print(f"Loading {DATASET_ID}/{CONFIG} (split={SPLIT})...") - ds = load_dataset(DATASET_ID, CONFIG, split=SPLIT) - print(f"Loaded {len(ds)} rows") - samples = [build_sample(dict(row)) for row in ds] - - done_ids = load_completed_ids(pred_path) - pending = [s for s in samples if s["id"] not in done_ids] - if limit is not None: - pending = pending[:limit] - print(f"--limit applied: will run at most {limit} sample(s)") - print(f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})") - if not pending: - return - - writer = JsonlWriter(pred_path) - semaphore = asyncio.Semaphore(CONCURRENCY) - progress = {"total": len(pending), "done": 0, "correct": 0, "failed": 0} - - tasks = [process_sample(s, semaphore, writer, progress) for s in pending] - try: - await tqdm_asyncio.gather(*tasks, desc=f"GPQA Diamond / {MODEL}") - except Exception: - traceback.print_exc() - acc = progress["correct"] / progress["done"] if progress["done"] else 0.0 - print(f"\nRun finished: {progress['done']}/{progress['total']} answered, " - f"{progress['correct']} correct (acc={acc:.4f}), {progress['failed']} failed.") - - -def run_evaluation(pred_path: Path, metrics_path: Path): - if not pred_path.exists(): - print(f"No predictions found at {pred_path}") - sys.exit(1) - results = load_records(pred_path) - if not results: - print(f"No records in {pred_path}") - sys.exit(1) - for r in results: - if r.get("prediction") is None and r.get("response"): - r["prediction"] = parse_letter(r["response"]) - if r.get("correct") is None and r.get("prediction") is not None: - r["correct"] = r["prediction"] == r.get("correct_letter") - - metrics = compute_metrics(results) - print_summary(metrics) - output = { - **metrics, - "dataset": DATASET_ID, - "config": CONFIG, - "split": SPLIT, - "model": MODEL, - "thinking": THINKING, - "effort": EFFORT, - "temperature": TEMPERATURE, - "concurrency": CONCURRENCY, - "provider": "openrouter", - } - metrics_path.parent.mkdir(parents=True, exist_ok=True) - with open(metrics_path, "w") as f: - json.dump(output, f, indent=2, ensure_ascii=False) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - global MODEL, THINKING, EFFORT - parser = argparse.ArgumentParser(description="GPQA Diamond benchmark via OpenRouter") - parser.add_argument("--model", default=DEFAULT_MODEL, - help="OpenRouter model id (e.g. x-ai/grok-4.3)") - parser.add_argument("--thinking", default=DEFAULT_THINKING, choices=["on", "off"], - help="Toggle reasoning via OpenRouter's `reasoning.enabled`") - parser.add_argument("--effort", default=DEFAULT_EFFORT, choices=["low","medium","high"], - help="OpenRouter `reasoning.effort` (low|medium|high). Only applied when thinking=on.") - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument("--limit", type=int, default=None, - help="Only run the first N unanswered samples") - args = parser.parse_args() - - MODEL = args.model - THINKING = args.thinking - EFFORT = args.effort - - effort_suffix = f"_effort{EFFORT}" if (THINKING == "on" and EFFORT) else "" - tag = f"{model_slug(MODEL)}_thinking{THINKING}{effort_suffix}_gpqa_diamond" - pred_path = RESULTS_DIR / f"{tag}_responses.jsonl" - metrics_path = RESULTS_DIR / f"{tag}_metrics.json" - - if args.evaluate_only: - run_evaluation(pred_path, metrics_path) - elif args.predict_only: - asyncio.run(run_predictions(pred_path, limit=args.limit)) - else: - asyncio.run(run_predictions(pred_path, limit=args.limit)) - run_evaluation(pred_path, metrics_path) - - -if __name__ == "__main__": - main() diff --git a/scripts/run_fireworks_suite.sh b/scripts/run_fireworks_suite.sh index 7e05c1f..fe5464b 100755 --- a/scripts/run_fireworks_suite.sh +++ b/scripts/run_fireworks_suite.sh @@ -92,13 +92,13 @@ run_step ocrbench uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks --model --provider "$PROVIDER" --reasoning "$BENCH_OPENROUTER_EFFORT" # Gated dataset — needs `hf auth login` plus accepted terms on the Hub. -# GPQA has separate per-host runners; gpqa_fireworks talks only to Fireworks, so -# an OpenRouter model id would 404 there. -if [ "$PROVIDER" = "openrouter" ]; then - run_step gpqa uv run -m benchmarks.gpqa.gpqa_openrouter --model "$MODEL" \ - --thinking "$([ "$REASONING" = "high" ] && echo on || echo off)" +# GPQA now runs through the unified CLI; provider/model/reasoning come from the +# target (bench_core/targets.yaml), so set TARGET= (see `list-targets`). +if [ -n "${TARGET:-}" ]; then + run_step gpqa uv run python -m bench_core run --benchmark gpqa --target "$TARGET" \ + --reasoning "$([ "$REASONING" = "high" ] && echo high || echo off)" else - run_step gpqa uv run -m benchmarks.gpqa.gpqa_fireworks --model "$MODEL" + echo " [skip] gpqa: set TARGET= (python -m bench_core list-targets)" fi # Grounding is scored strictly above; this adds the repo's uniform diff --git a/tests/test_gpqa_bench.py b/tests/test_gpqa_bench.py new file mode 100644 index 0000000..6e6db8a --- /dev/null +++ b/tests/test_gpqa_bench.py @@ -0,0 +1,85 @@ +"""GPQA benchmark: parse unit tests + offline scoring-parity replay. + +The parity test is the real migration gate: replay each archived responses.jsonl +through the NEW parse/score and assert the score is bit-identical to the archived +metrics.json — proving the port preserves scores with zero API spend. +""" + +import json +import pathlib +from types import SimpleNamespace as NS + +import pytest + +from benchmarks.gpqa import bench + +RESULTS = pathlib.Path(__file__).resolve().parent.parent / "results" +ARCHIVED_TAGS = [ + "fireworks_inkling_thinkingoff_gpqa_diamond", + "fireworks_deepseekv4flash_thinkingoff_gpqa_diamond", + "gemini37flash_thinkingdefault_gpqa_diamond", + "thinkingmachinesinklingsmall_thinkingon_gpqa_diamond", +] + + +def _resp(text): + return NS(text=text) + + +def test_parse_single_letter(): + assert bench.parse(_resp("A"), None) == "A" + assert bench.parse(_resp("d"), None) == "D" + + +def test_parse_letter_in_sentence(): + assert bench.parse(_resp("The answer is B."), None) == "B" + assert bench.parse(_resp("Answer: C"), None) == "C" + + +def test_parse_empty_is_none(): + assert bench.parse(_resp(""), None) is None + assert bench.parse(_resp(" "), None) is None + + +def test_score_joins_records_to_samples(): + samples = [ + {"id": "1", "correct_letter": "A", "domain": "Physics"}, + {"id": "2", "correct_letter": "B", "domain": "Chemistry"}, + ] + records = [{"id": "1", "prediction": "A"}, {"id": "2", "prediction": "C"}] + m = bench.score(records, samples) + assert m["correct"] == 1 + assert m["total"] == 2 + assert m["accuracy"] == 0.5 + assert m["per_domain"]["Physics"]["accuracy"] == 1.0 + + +@pytest.mark.parametrize("tag", ARCHIVED_TAGS) +def test_scoring_parity_with_archived_run(tag): + resp_path = RESULTS / f"{tag}_responses.jsonl" + metrics_path = RESULTS / f"{tag}_metrics.json" + if not (resp_path.exists() and metrics_path.exists()): + pytest.skip(f"archived run {tag} not present") + + archived = json.loads(metrics_path.read_text()) + samples, records = [], [] + for line in resp_path.read_text().splitlines(): + if not line.strip(): + continue + rec = json.loads(line) + samples.append( + { + "id": rec["id"], + "correct_letter": rec["correct_letter"], + "domain": rec.get("domain"), + } + ) + records.append( + {"id": rec["id"], "prediction": bench.parse(_resp(rec["response"]), None)} + ) + + m = bench.score(records, samples) + assert m["total"] == archived["total"] + assert m["correct"] == archived["correct"] + assert m["accuracy"] == archived["accuracy"] + assert m["unparseable"] == archived["unparseable"] From 70dcdcd0fb43879cf567679b7dfcd460cc92f682 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 08:29:26 +0530 Subject: [PATCH 08/38] feat(bench_core): migrate ASR (VoxPopuli) onto the abstraction; retire 2 forks benchmarks/asr/bench.py (audio fetch + WER/CER scoring) replaces voxpopuli_aa.py and voxpopuli_aa_multi.py; the per-provider audio wire-shape (input_audio / audio_url / gemini_part / file_block) is now handled by the adapters, fixing the silent audio_url->OpenRouter drop by construction. Offline parity test replays the 4 archived ASR runs through the new score and matches corpus/time-weighted WER + CER (inkling 0.0461, gemini 0.0358, inkling-small 0.0409). CLI gains --benchmark asr and a generic primary-metric summary. Suite + README updated. 125 tests pass. --- README.md | 16 +- bench_core/cli.py | 12 +- benchmarks/asr/bench.py | 136 +++++++++ benchmarks/asr/voxpopuli_aa.py | 406 -------------------------- benchmarks/asr/voxpopuli_aa_multi.py | 421 --------------------------- benchmarks/gpqa/bench.py | 1 + scripts/run_fireworks_suite.sh | 8 +- tests/test_asr_bench.py | 76 +++++ 8 files changed, 230 insertions(+), 846 deletions(-) create mode 100644 benchmarks/asr/bench.py delete mode 100644 benchmarks/asr/voxpopuli_aa.py delete mode 100644 benchmarks/asr/voxpopuli_aa_multi.py create mode 100644 tests/test_asr_bench.py diff --git a/README.md b/README.md index 3a6eaa8..58a1726 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ uv run -m benchmarks.mmmlu.mmmlu_multi --provider fireworks --model accou uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider fireworks --model accounts/fireworks/models/inkling --setting standard uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider fireworks --model accounts/fireworks/models/inkling --setting vision uv run -m benchmarks.obj_detection.refcoco_multi --provider fireworks --model accounts/fireworks/models/inkling -uv run -m benchmarks.asr.voxpopuli_aa_multi --provider fireworks --model accounts/fireworks/models/inkling +uv run python -m bench_core run --target inkling --benchmark asr uv run -m benchmarks.spider2_lite.spider2_lite --provider fireworks ``` @@ -216,16 +216,10 @@ Links: [dataset](https://huggingface.co/datasets/ArtificialAnalysis/VoxPopuli-Cl Metric: WER with Whisper-style text normalization. ```bash -# Interfaze -uv run -m benchmarks.asr.voxpopuli_aa - -# Other providers (audio-capable) -uv run -m benchmarks.asr.voxpopuli_aa_multi --provider gemini --model gemini-3-flash-preview -uv run -m benchmarks.asr.voxpopuli_aa_multi --provider openai --model gpt-4o-audio-preview -uv run -m benchmarks.asr.voxpopuli_aa_multi --provider anthropic --model claude-sonnet-4-6 - -# Evaluate only -uv run -m benchmarks.asr.voxpopuli_aa --evaluate-only +# Any audio-capable target (the right audio wire-shape is chosen per provider) +uv run python -m bench_core run --target interfaze-beta --benchmark asr +uv run python -m bench_core run --target gemini-3.7-flash --benchmark asr +uv run python -m bench_core run --target inkling --benchmark asr ``` --- diff --git a/bench_core/cli.py b/bench_core/cli.py index 199a025..2b080d5 100644 --- a/bench_core/cli.py +++ b/bench_core/cli.py @@ -18,6 +18,7 @@ # benchmark short-name -> module exposing NAME/load_samples/build_request/parse/score BENCHMARKS = { "gpqa": "benchmarks.gpqa.bench", + "asr": "benchmarks.asr.bench", } @@ -67,7 +68,7 @@ def cmd_run(args) -> None: "target": target.name, "provider": target.provider, "model_id": target.model_id, - "n": metrics.get("total"), + "n": metrics.get("total") or metrics.get("num_samples"), "reasoning": { "mode": mode, "style": caps.reasoning.style.value, @@ -88,11 +89,12 @@ def cmd_run(args) -> None: } ) - acc = metrics.get("accuracy") + primary = getattr(bench, "PRIMARY_METRIC", None) + pv = metrics.get(primary) if primary else None + head = f"{primary}={pv:.4f} " if isinstance(pv, (int, float)) else "" print( - f"\n{bench.NAME} / {target.name}: " - + (f"accuracy={acc:.4f} " if acc is not None else "") - + f"({metrics.get('correct')}/{metrics.get('total')}) failed={result.n_failed}" + f"\n{bench.NAME} / {target.name}: {head}" + f"n={metrics.get('n')} failed={result.n_failed}" ) print(f"written to {store.metrics_path}") if result.hints: diff --git a/benchmarks/asr/bench.py b/benchmarks/asr/bench.py new file mode 100644 index 0000000..3780e8e --- /dev/null +++ b/benchmarks/asr/bench.py @@ -0,0 +1,136 @@ +"""VoxPopuli-Cleaned-AA ASR: transcribe audio, score corpus + time-weighted WER.""" + +from __future__ import annotations + +import re +import unicodedata + +from jiwer import cer, wer + +from bench_core.request import AudioPart, Message, ReasoningSpec, Request, TextPart + +NAME = "voxpopuli_aa" +ID_KEY = "id" +PRIMARY_METRIC = "corpus_wer" +DEFAULTS = {"reasoning": "off", "rate_limit": 25, "max_in_flight": 8} + +_DATASET_ID = "ArtificialAnalysis/VoxPopuli-Cleaned-AA" +_SPLIT = "test" + +PROMPT = ( + "Transcribe the following audio. Fix anything that needs fixing — " + "disfluencies, stutters, obvious misspeaks, garbled words, or misheard " + "named entities — so the transcription reads as the speaker clearly " + "intended. Output ONLY the cleaned transcription, no commentary, labels, " + "speaker tags, or timestamps." +) + +_NON_ALNUM_SPACE = re.compile(r"[^a-z0-9' ]+") +_WHITESPACE = re.compile(r"\s+") + + +def normalize_text(text: str) -> str: + # Whisper-style: NFKC, lowercase, strip punctuation (keep apostrophes), collapse ws. + if not text: + return "" + text = unicodedata.normalize("NFKC", text).lower() + text = _NON_ALNUM_SPACE.sub(" ", text) + return _WHITESPACE.sub(" ", text).strip() + + +def load_samples(sample_size: int | None = None) -> list[dict]: + from datasets import load_dataset + from huggingface_hub import hf_hub_download + from tqdm import tqdm + + ds = load_dataset(_DATASET_ID, split=_SPLIT) + rows = list(ds)[:sample_size] if sample_size else list(ds) + samples = [] + for row in tqdm(rows, desc="fetch audio"): + row = dict(row) + # Warm the HF cache; build_request reads the local file (fast, no memory held). + path = hf_hub_download( + repo_id=_DATASET_ID, + repo_type="dataset", + filename=f"audio/{row['file_name']}", + ) + samples.append( + { + "id": str(row["id"]), + "file_name": row["file_name"], + "audio_path": path, + "transcript": row["transcript"], + "duration": row.get("duration"), + "language": row.get("language"), + } + ) + return samples + + +def build_request(sample: dict, mode: str) -> Request: + with open(sample["audio_path"], "rb") as f: + audio = f.read() + return Request( + messages=[Message("user", [TextPart(PROMPT), AudioPart(audio, "audio/wav")])], + reasoning=ReasoningSpec(mode), + temperature=0.0, + ) + + +def parse(response, sample) -> str: + return (response.text or "").strip() + + +def _sample_metric(fn, gt_norm: str, hyp_norm: str) -> float: + if not gt_norm: + return float("inf") + try: + return float(fn(gt_norm, hyp_norm)) + except (ValueError, ZeroDivisionError): + return float("inf") + + +def score(records: list[dict], samples: list[dict]) -> dict: + by_id = {s["id"]: s for s in samples} + rows = [] + for r in records: + s = by_id.get(r["id"]) + if s is None: + continue + gt = normalize_text(s["transcript"]) + hyp = normalize_text(r.get("prediction") or "") + rows.append( + { + "gt": gt, + "hyp": hyp, + "wer": _sample_metric(wer, gt, hyp), + "cer": _sample_metric(cer, gt, hyp), + "duration": s.get("duration"), + } + ) + if not rows: + return {} + + refs = [x["gt"] for x in rows if x["gt"]] + hyps = [x["hyp"] for x in rows if x["gt"]] + total_dur = sum(x["duration"] or 0 for x in rows) + finite_wer = [x["wer"] for x in rows if x["wer"] != float("inf")] + finite_cer = [x["cer"] for x in rows if x["cer"] != float("inf")] + + return { + "corpus_wer": float(wer(refs, hyps)) if refs else float("inf"), + "corpus_cer": float(cer(refs, hyps)) if refs else float("inf"), + "mean_sample_wer": sum(finite_wer) / max(1, len(rows)), + "mean_sample_cer": sum(finite_cer) / max(1, len(rows)), + "time_weighted_wer": ( + sum( + (x["wer"] if x["wer"] != float("inf") else 0) * (x["duration"] or 0) + for x in rows + ) + / total_dur + if total_dur > 0 + else float("inf") + ), + "num_samples": len(rows), + "total_duration_s": total_dur, + } diff --git a/benchmarks/asr/voxpopuli_aa.py b/benchmarks/asr/voxpopuli_aa.py deleted file mode 100644 index e3127bf..0000000 --- a/benchmarks/asr/voxpopuli_aa.py +++ /dev/null @@ -1,406 +0,0 @@ -""" -VoxPopuli-Cleaned-AA ASR benchmark for Interfaze. - -628 English speech samples drawn from European Parliament recordings, with -re-cleaned ground-truth transcripts curated by Artificial Analysis. This is -one of the three datasets that make up AA-WER v2.0 (the ASR benchmark Gemini 3 -Pro and other frontier audio models report on). The other two are -Earnings22-Cleaned-AA (open, 6 samples) and AA-AgentTalk (proprietary). - -Dataset: https://huggingface.co/datasets/ArtificialAnalysis/VoxPopuli-Cleaned-AA - -Metric: WER (Word Error Rate) with Whisper-style text normalization — - lowercase, strip punctuation, NFKC, collapse whitespace. - -Usage: - uv run -m benchmarks.asr.voxpopuli_aa - uv run -m benchmarks.asr.voxpopuli_aa --limit 5 - uv run -m benchmarks.asr.voxpopuli_aa --evaluate-only -""" - -import sys -import os -import re -import json -import time -import base64 -import asyncio -import argparse -import traceback -import unicodedata -from pathlib import Path - -from datasets import load_dataset -from huggingface_hub import hf_hub_download -from jiwer import wer, cer -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -from src.commons import invoke_interfaze # noqa: E402 - -RESULTS_DIR = PROJECT_ROOT / "results" -DATASET_ID = "ArtificialAnalysis/VoxPopuli-Cleaned-AA" -SPLIT = "test" -REASONING_EFFORT = None # off -TEMPERATURE = 0.0 # deterministic -RATE_LIMIT = 25 -MAX_RETRIES = 3 - -PROMPT = ( - "Transcribe the following audio. Fix anything that needs fixing — " - "disfluencies, stutters, obvious misspeaks, garbled words, or misheard " - "named entities — so the transcription reads as the speaker clearly " - "intended. Output ONLY the cleaned transcription, no commentary, labels, " - "speaker tags, or timestamps." -) - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -class JsonlWriter: - def __init__(self, path: Path): - self.path = path - self.path.parent.mkdir(parents=True, exist_ok=True) - self._lock = asyncio.Lock() - - async def append(self, record: dict): - line = json.dumps(record, ensure_ascii=False) - async with self._lock: - with open(self.path, "a", encoding="utf-8") as f: - f.write(line + "\n") - f.flush() - os.fsync(f.fileno()) - - -def load_completed_ids(path: Path) -> set[str]: - if not path.exists(): - return set() - done: set[str] = set() - with open(path, encoding="utf-8") as f: - for line_no, line in enumerate(f, 1): - line = line.strip() - if not line: - continue - try: - rec = json.loads(line) - except json.JSONDecodeError: - tqdm.write(f"[resume] skipping malformed line {line_no} in {path}") - continue - if rec.get("response"): - done.add(str(rec["id"])) - return done - - -def load_records(path: Path) -> list[dict]: - records: list[dict] = [] - if not path.exists(): - return records - with open(path, encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - records.append(json.loads(line)) - except json.JSONDecodeError: - continue - by_id: dict[str, dict] = {} - for r in records: - by_id[str(r["id"])] = r - return list(by_id.values()) - - -_NON_ALNUM_SPACE = re.compile(r"[^a-z0-9' ]+") -_WHITESPACE = re.compile(r"\s+") - - -def normalize_text(text: str) -> str: - """Whisper-style light normalization for WER: NFKC, lowercase, strip - punctuation (keeping apostrophes for contractions), collapse whitespace.""" - if not text: - return "" - text = unicodedata.normalize("NFKC", text) - text = text.lower() - text = _NON_ALNUM_SPACE.sub(" ", text) - text = _WHITESPACE.sub(" ", text).strip() - return text - - -def fetch_audio_bytes(file_name: str) -> bytes: - """Pull a single audio/*.wav from the HF dataset repo (cached locally).""" - path = hf_hub_download( - repo_id=DATASET_ID, repo_type="dataset", - filename=f"audio/{file_name}", - ) - with open(path, "rb") as f: - return f.read() - - -def build_sample(row: dict) -> dict: - return { - "id": str(row["id"]), - "file_name": row["file_name"], - "transcript": row["transcript"], - "duration": row.get("duration"), - "gender": row.get("gender"), - "language": row.get("language"), - } - - -async def process_sample(sample: dict, rate_limiter, writer: JsonlWriter, - progress: dict) -> dict | None: - last_error: str | None = None - - # Fetch audio once (outside the retry loop — it's cached by hf_hub_download). - try: - audio_bytes = await asyncio.to_thread(fetch_audio_bytes, sample["file_name"]) - except Exception as e: - tqdm.write(f"[fetch error] id={sample['id']}: {type(e).__name__}: {e}") - progress["failed"] += 1 - return None - - b64 = base64.b64encode(audio_bytes).decode("ascii") - data_url = f"data:audio/wav;base64,{b64}" - - messages = [{ - "role": "user", - "content": [ - {"type": "text", "text": PROMPT}, - {"type": "file", "file": {"filename": sample["file_name"], "file_data": data_url}}, - ], - }] - - for attempt in range(1, MAX_RETRIES + 1): - await rate_limiter.acquire() - start = time.perf_counter() - try: - response = await asyncio.to_thread( - invoke_interfaze, - messages, - reasoning_effort=REASONING_EFFORT, - temperature=TEMPERATURE, - ) - latency_ms = int((time.perf_counter() - start) * 1000) - content = (response.choices[0].message.content or "").strip() - request_id = getattr(response, "id", None) - if not content: - last_error = "empty response content" - raise RuntimeError(last_error) - - pred_norm = normalize_text(content) - gt_norm = normalize_text(sample["transcript"]) - try: - sample_wer = float(wer(gt_norm, pred_norm)) if gt_norm else float("inf") - except Exception: - sample_wer = float("inf") - try: - sample_cer = float(cer(gt_norm, pred_norm)) if gt_norm else float("inf") - except Exception: - sample_cer = float("inf") - - record = { - "id": sample["id"], - "file_name": sample["file_name"], - "duration": sample["duration"], - "gender": sample["gender"], - "language": sample["language"], - "transcript": sample["transcript"], - "transcript_normalized": gt_norm, - "prediction": content, - "prediction_normalized": pred_norm, - "wer": sample_wer, - "cer": sample_cer, - "response": content, - "request_id": request_id, - "latency_ms": latency_ms, - "attempts": attempt, - } - await writer.append(record) - - progress["done"] += 1 - progress["sum_wer"] += sample_wer if sample_wer != float("inf") else 0 - tqdm.write( - f"[{progress['done']}/{progress['total']}] WER={sample_wer:.3f} " - f"CER={sample_cer:.3f} id={sample['id']} dur={sample['duration']}s " - f"latency={latency_ms}ms req_id={request_id}" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - tqdm.write( - f"[error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(2 ** (attempt - 1)) - - progress["failed"] += 1 - tqdm.write(f"[FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") - return None - - -def compute_metrics(results: list[dict]) -> dict: - if not results: - return {} - # Aggregate WER/CER the correct way: per-ref length weighting — not a simple - # mean of per-sample WERs (long clips shouldn't count the same as short ones). - refs = [r["transcript_normalized"] for r in results if r.get("transcript_normalized")] - hyps = [r["prediction_normalized"] for r in results if r.get("transcript_normalized")] - corpus_wer = float(wer(refs, hyps)) if refs else float("inf") - corpus_cer = float(cer(refs, hyps)) if refs else float("inf") - - # Simple per-sample mean for reference. - mean_wer = sum(r["wer"] for r in results if r["wer"] != float("inf")) / max(1, len(results)) - mean_cer = sum(r["cer"] for r in results if r["cer"] != float("inf")) / max(1, len(results)) - - # Time-weighted WER (the AA-WER convention within a dataset) - total_dur = sum(r.get("duration", 0) or 0 for r in results) - time_weighted_wer = ( - sum((r["wer"] if r["wer"] != float("inf") else 0) * (r.get("duration", 0) or 0) - for r in results) / total_dur if total_dur > 0 else float("inf") - ) - - latencies = [r["latency_ms"] for r in results if isinstance(r.get("latency_ms"), int)] - latency_stats = {} - if latencies: - lats = sorted(latencies) - n = len(lats) - latency_stats = { - "count": n, "mean_ms": sum(lats) / n, - "p50_ms": lats[n // 2], - "p90_ms": lats[min(n - 1, int(n * 0.9))], - "p99_ms": lats[min(n - 1, int(n * 0.99))], - "max_ms": lats[-1], - } - - return { - "corpus_wer": corpus_wer, - "corpus_cer": corpus_cer, - "mean_sample_wer": mean_wer, - "mean_sample_cer": mean_cer, - "time_weighted_wer": time_weighted_wer, - "num_samples": len(results), - "total_duration_s": total_dur, - "latency": latency_stats, - } - - -def print_summary(metrics: dict): - print(f"\n{'=' * 60}") - print(f"VoxPopuli-Cleaned-AA Results (Interfaze, reasoning={REASONING_EFFORT}, temp={TEMPERATURE})") - print(f"{'=' * 60}") - print(f"Samples : {metrics['num_samples']}") - print(f"Total audio duration : {metrics['total_duration_s']:.1f}s") - print(f"Corpus WER : {metrics['corpus_wer']:.4f} ← primary metric") - print(f"Corpus CER : {metrics['corpus_cer']:.4f}") - print(f"Time-weighted WER : {metrics['time_weighted_wer']:.4f} ← AA-WER convention") - print(f"Mean per-sample WER : {metrics['mean_sample_wer']:.4f}") - print(f"Mean per-sample CER : {metrics['mean_sample_cer']:.4f}") - if metrics.get("latency"): - lat = metrics["latency"] - print(f"Latency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " - f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms max={lat['max_ms']}ms") - - -async def run_predictions(pred_path: Path, limit: int | None = None): - print(f"Loading {DATASET_ID}, split={SPLIT}...") - dataset = load_dataset(DATASET_ID, split=SPLIT) - print(f"Loaded {len(dataset)} samples") - - samples = [build_sample(dict(row)) for row in dataset] - done_ids = load_completed_ids(pred_path) - pending = [s for s in samples if s["id"] not in done_ids] - if limit is not None: - pending = pending[:limit] - print(f"--limit applied: will run at most {limit} sample(s)") - print(f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})") - if not pending: - return - - writer = JsonlWriter(pred_path) - rate_limiter = RateLimiter(RATE_LIMIT) - progress = {"total": len(pending), "done": 0, "failed": 0, "sum_wer": 0.0} - - tasks = [process_sample(s, rate_limiter, writer, progress) for s in pending] - try: - await tqdm_asyncio.gather(*tasks, desc=f"{DATASET_ID.split('/')[-1]}/{SPLIT}") - except Exception: - traceback.print_exc() - print(f"\nRun finished: {progress['done']}/{progress['total']} answered, " - f"{progress['failed']} failed.") - - -def run_evaluation(pred_path: Path, metrics_path: Path): - if not pred_path.exists(): - print(f"No predictions found at {pred_path}") - sys.exit(1) - results = load_records(pred_path) - if not results: - print(f"No records in {pred_path}") - sys.exit(1) - - for r in results: - if r.get("prediction_normalized") is None and r.get("response"): - r["prediction_normalized"] = normalize_text(r["response"]) - if r.get("transcript_normalized") is None and r.get("transcript"): - r["transcript_normalized"] = normalize_text(r["transcript"]) - - metrics = compute_metrics(results) - print_summary(metrics) - output = { - **metrics, - "dataset": DATASET_ID, "split": SPLIT, - "reasoning_effort": REASONING_EFFORT, "temperature": TEMPERATURE, - "rate_limit": RATE_LIMIT, "model": "interfaze-beta", - } - metrics_path.parent.mkdir(parents=True, exist_ok=True) - with open(metrics_path, "w") as f: - json.dump(output, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - parser = argparse.ArgumentParser(description="VoxPopuli-Cleaned-AA ASR benchmark") - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument("--limit", type=int, default=None) - args = parser.parse_args() - - pred_path = RESULTS_DIR / "voxpopuli_aa_responses.jsonl" - metrics_path = RESULTS_DIR / "voxpopuli_aa_metrics.json" - - if args.evaluate_only: - run_evaluation(pred_path, metrics_path) - elif args.predict_only: - asyncio.run(run_predictions(pred_path, limit=args.limit)) - else: - asyncio.run(run_predictions(pred_path, limit=args.limit)) - run_evaluation(pred_path, metrics_path) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/asr/voxpopuli_aa_multi.py b/benchmarks/asr/voxpopuli_aa_multi.py deleted file mode 100644 index 1ea2f09..0000000 --- a/benchmarks/asr/voxpopuli_aa_multi.py +++ /dev/null @@ -1,421 +0,0 @@ -""" -VoxPopuli-Cleaned-AA ASR benchmark — multi-provider edition. - -Runs the same prompt + dataset + scoring as benchmarks.asr.voxpopuli_aa, but -against non-interfaze providers (Gemini, Anthropic, OpenAI) for head-to-head -WER comparison. Designed to run in parallel with the interfaze run without -stepping on its checkpoint file. - -Output: results/voxpopuli_aa___responses.jsonl - -Usage: - uv run -m benchmarks.asr.voxpopuli_aa_multi --provider gemini --model gemini-3-flash-preview - uv run -m benchmarks.asr.voxpopuli_aa_multi --provider gemini --model gemini-3-flash-preview --limit 5 -""" - -import os -import sys -import json -import time -import base64 -import asyncio -import argparse -import traceback -from pathlib import Path - -from datasets import load_dataset -from dotenv import load_dotenv -from jiwer import wer, cer -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -load_dotenv() - -# reasoning.effort sent to OpenRouter-hosted models. Override per run. -OPENROUTER_EFFORT = os.getenv("BENCH_OPENROUTER_EFFORT", "none") - -def _gemini_floor(model: str) -> str: - """Lowest thinking_level the given Gemini model actually accepts. - - gemini-3.7+ flash rejects MINIMAL server-side ("Thinking level MINIMAL is - not supported for this model"), and the SDK enum has no "off"/"none", so - "low" is its true floor. Older 3.x flash models still take "minimal". - BENCH_GEMINI_FLOOR overrides for anything not covered here. - """ - override = os.getenv("BENCH_GEMINI_FLOOR") - if override: - return override - m = model.lower() - if any(v in m for v in ("3.7", "3.8", "3.9", "4.")): - return "low" - return "minimal" - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -# Reuse everything shared: prompt, WER normalization, JSONL writer, etc. -from benchmarks.asr.voxpopuli_aa import ( # noqa: E402 - PROMPT, - DATASET_ID, - SPLIT, - RATE_LIMIT, - MAX_RETRIES, - RateLimiter, - JsonlWriter, - normalize_text, - fetch_audio_bytes, - build_sample, - load_completed_ids, - load_records, - compute_metrics, - print_summary, -) - -RESULTS_DIR = PROJECT_ROOT / "results" - - -def _load_interfaze_env() -> dict: - """Optional key source — the project .env (loaded above) covers providers - keyed there, so a missing file is not fatal.""" - path = Path.home() / "interfaze" / ".env.local" - if not path.exists(): - return {} - env = {} - for line in path.read_text().splitlines(): - line = line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - k, v = line.split("=", 1) - env[k.strip()] = v.strip().strip('"').strip("'") - return env - - -# -------- provider adapters: (audio_bytes, prompt, model, client) -> (content, req_id) -------- - - -def call_gemini(audio_bytes: bytes, prompt_text: str, model: str, client): - """Thinking OFF (or as close as the model allows): - - Gemini 3.x Pro: thinking_level="low" (Pro rejects 'minimal'; min is 'low'). - - Gemini 3.x Flash: thinking_level="minimal" (true disable not supported). - - Gemini 2.5 Pro: thinking_budget=128 (Pro can't go lower than 128). - - Gemini 2.5 Flash: thinking_budget=0 (true disable). - Input audio as inline bytes (Gemini accepts up to ~20 MB inline).""" - from google.genai import types - - m = model.lower() - if m.startswith("gemini-2.5-pro"): - thinking = types.ThinkingConfig(thinking_budget=128) - elif m.startswith("gemini-2.5-flash"): - thinking = types.ThinkingConfig(thinking_budget=0) - elif "pro" in m: - thinking = types.ThinkingConfig(thinking_level="low") - else: - thinking = types.ThinkingConfig(thinking_level=_gemini_floor(model)) - config = types.GenerateContentConfig( - thinking_config=thinking, - temperature=0.0, - ) - resp = client.models.generate_content( - model=model, - contents=[ - types.Part.from_bytes(data=audio_bytes, mime_type="audio/wav"), - prompt_text, - ], - config=config, - ) - content = (resp.text or "").strip() - request_id = getattr(resp, "response_id", None) or "" - return content, request_id - - -def call_fireworks(audio_bytes: bytes, prompt_text: str, model: str, client): - """Fireworks-hosted models (Inkling) take audio as an `audio_url` block with - a data URL — the OpenAI `input_audio` shape is rejected by this endpoint. - Inkling always thinks; "none" is the reasoning floor.""" - b64 = base64.b64encode(audio_bytes).decode("utf-8") - resp = client.chat.completions.create( - model=model, - messages=[ - { - "role": "user", - "content": [ - {"type": "text", "text": prompt_text}, - { - "type": "audio_url", - "audio_url": {"url": f"data:audio/wav;base64,{b64}"}, - }, - ], - } - ], - temperature=0.0, - reasoning_effort="none", - ) - content = (resp.choices[0].message.content or "").strip() - return content, getattr(resp, "id", "") or "" - - -def call_openrouter(audio_bytes: bytes, prompt_text: str, model: str, client): - """OpenRouter-hosted audio models. Same audio_url data-URL shape as Fireworks; - reasoning goes through extra_body. See refcoco_multi.call_openrouter for the - host-vs-host caveat on effort="none".""" - b64 = base64.b64encode(audio_bytes).decode("utf-8") - resp = client.chat.completions.create( - model=model, - messages=[ - { - "role": "user", - "content": [ - {"type": "text", "text": prompt_text}, - # OpenRouter needs OpenAI's `input_audio` block. The - # `audio_url` data-URI shape that Fireworks requires is - # ACCEPTED here with no error but the audio is silently - # dropped — the model then hallucinates a fluent transcript - # (measured: exact match with input_audio vs unrelated text - # with audio_url, i.e. WER ~1.0 across the whole set). - { - "type": "input_audio", - "input_audio": {"data": b64, "format": "wav"}, - }, - ], - } - ], - temperature=0.0, - extra_body={"reasoning": {"effort": OPENROUTER_EFFORT}}, - ) - return (resp.choices[0].message.content or "").strip(), getattr(resp, "id", "") or "" - - -def build_client(provider: str, env: dict): - if provider == "gemini": - from google import genai - - api_key = ( - env.get("GEMINI_KEY") - or os.getenv("GEMINI_API_KEY") - or os.getenv("GEMINI_KEY") - or os.getenv("GOOGLE_API_KEY") - ) - if not api_key: - raise RuntimeError("GEMINI_API_KEY missing from .env") - return genai.Client(api_key=api_key) - if provider == "openrouter": - from openai import OpenAI - - api_key = ( - env.get("OPENROUTER_API_KEY") - or os.getenv("OPENROUTER_API_KEY") - or os.getenv("openrouter_api_key") - ) - if not api_key: - raise RuntimeError("OPENROUTER_API_KEY missing from .env") - return OpenAI(base_url="https://openrouter.ai/api/v1", api_key=api_key) - if provider == "fireworks": - from openai import OpenAI - - # .env in this repo may spell the key lowercase; accept either casing. - api_key = ( - env.get("FIREWORKS_API_KEY") - or os.getenv("FIREWORKS_API_KEY") - or os.getenv("fireworks_api_key") - ) - if not api_key: - raise RuntimeError("FIREWORKS_API_KEY missing from .env") - return OpenAI(base_url="https://api.fireworks.ai/inference/v1", api_key=api_key) - raise ValueError(f"Provider not yet supported here: {provider}") - - -def get_call_fn(provider: str): - return { - "gemini": call_gemini, - "fireworks": call_fireworks, - "openrouter": call_openrouter, - }[provider] - - -# -------- pipeline (mirrors voxpopuli_aa.process_sample, but routed via adapter) -------- - - -async def process_sample( - sample: dict, - call_fn, - model: str, - rate_limiter, - writer: JsonlWriter, - progress: dict, - provider: str, - client, -) -> dict | None: - last_error: str | None = None - - try: - audio_bytes = await asyncio.to_thread(fetch_audio_bytes, sample["file_name"]) - except Exception as e: - tqdm.write( - f"[{provider} fetch error] id={sample['id']}: {type(e).__name__}: {e}" - ) - progress["failed"] += 1 - return None - - for attempt in range(1, MAX_RETRIES + 1): - await rate_limiter.acquire() - start = time.perf_counter() - try: - content, request_id = await asyncio.to_thread( - call_fn, audio_bytes, PROMPT, model, client - ) - latency_ms = int((time.perf_counter() - start) * 1000) - if not content: - last_error = "empty response content" - raise RuntimeError(last_error) - - pred_norm = normalize_text(content) - gt_norm = normalize_text(sample["transcript"]) - sample_wer = float(wer(gt_norm, pred_norm)) if gt_norm else float("inf") - sample_cer = float(cer(gt_norm, pred_norm)) if gt_norm else float("inf") - - record = { - "id": sample["id"], - "file_name": sample["file_name"], - "duration": sample["duration"], - "gender": sample["gender"], - "language": sample["language"], - "transcript": sample["transcript"], - "transcript_normalized": gt_norm, - "prediction": content, - "prediction_normalized": pred_norm, - "wer": sample_wer, - "cer": sample_cer, - "response": content, - "request_id": request_id, - "latency_ms": latency_ms, - "attempts": attempt, - "provider": provider, - "model": model, - } - await writer.append(record) - - progress["done"] += 1 - tqdm.write( - f"[{provider} {progress['done']}/{progress['total']}] WER={sample_wer:.3f} " - f"CER={sample_cer:.3f} id={sample['id']} dur={sample['duration']}s " - f"latency={latency_ms}ms req_id={request_id}" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - tqdm.write( - f"[{provider} error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(2 ** (attempt - 1)) - - progress["failed"] += 1 - tqdm.write( - f"[{provider} FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}" - ) - return None - - -def build_tag(provider: str, model: str) -> str: - return f"voxpopuli_aa_{provider}_{model.replace('/', '_').replace(':', '_')}" - - -async def run(provider: str, model: str, pred_path: Path, limit: int | None): - env = _load_interfaze_env() - client = build_client(provider, env) - call_fn = get_call_fn(provider) - - print(f"[{provider}/{model}] Loading {DATASET_ID}, split={SPLIT}...") - dataset = load_dataset(DATASET_ID, split=SPLIT) - samples = [build_sample(dict(row)) for row in dataset] - - done_ids = load_completed_ids(pred_path) - pending = [s for s in samples if s["id"] not in done_ids] - if limit is not None: - pending = pending[:limit] - print(f"--limit applied: will run at most {limit} sample(s)") - print( - f"Resume: {len(done_ids)} completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})" - ) - if not pending: - return - - writer = JsonlWriter(pred_path) - rate_limiter = RateLimiter(RATE_LIMIT) - progress = {"total": len(pending), "done": 0, "failed": 0} - - tasks = [ - process_sample( - s, call_fn, model, rate_limiter, writer, progress, provider, client - ) - for s in pending - ] - try: - await tqdm_asyncio.gather(*tasks, desc=f"{provider}/{model}") - except Exception: - traceback.print_exc() - print( - f"\n[{provider}/{model}] Run finished: {progress['done']}/{progress['total']}, " - f"{progress['failed']} failed." - ) - - -def run_evaluation(pred_path: Path, metrics_path: Path, provider: str, model: str): - if not pred_path.exists(): - print(f"No predictions found at {pred_path}") - sys.exit(1) - results = load_records(pred_path) - if not results: - print(f"No records in {pred_path}") - sys.exit(1) - for r in results: - if r.get("prediction_normalized") is None and r.get("response"): - r["prediction_normalized"] = normalize_text(r["response"]) - if r.get("transcript_normalized") is None and r.get("transcript"): - r["transcript_normalized"] = normalize_text(r["transcript"]) - metrics = compute_metrics(results) - print_summary(metrics) - output = { - **metrics, - "dataset": DATASET_ID, - "split": SPLIT, - "provider": provider, - "model": model, - "rate_limit": RATE_LIMIT, - } - metrics_path.parent.mkdir(parents=True, exist_ok=True) - with open(metrics_path, "w") as f: - json.dump(output, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - parser = argparse.ArgumentParser( - description="Multi-provider VoxPopuli-Cleaned-AA eval" - ) - parser.add_argument("--provider", required=True, choices=["gemini", "fireworks", "openrouter"]) - parser.add_argument("--model", required=True) - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument("--limit", type=int, default=None) - args = parser.parse_args() - - tag = build_tag(args.provider, args.model) - pred_path = RESULTS_DIR / f"{tag}_responses.jsonl" - metrics_path = RESULTS_DIR / f"{tag}_metrics.json" - - if args.evaluate_only: - run_evaluation(pred_path, metrics_path, args.provider, args.model) - elif args.predict_only: - asyncio.run(run(args.provider, args.model, pred_path, limit=args.limit)) - else: - asyncio.run(run(args.provider, args.model, pred_path, limit=args.limit)) - run_evaluation(pred_path, metrics_path, args.provider, args.model) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/gpqa/bench.py b/benchmarks/gpqa/bench.py index 2a5f2fe..b9ed208 100644 --- a/benchmarks/gpqa/bench.py +++ b/benchmarks/gpqa/bench.py @@ -10,6 +10,7 @@ NAME = "gpqa" ID_KEY = "id" +PRIMARY_METRIC = "accuracy" DEFAULTS = {"reasoning": "off", "rate_limit": 10, "max_in_flight": 8} _DATASET_ID = "Idavidrein/gpqa" diff --git a/scripts/run_fireworks_suite.sh b/scripts/run_fireworks_suite.sh index fe5464b..26c81fe 100755 --- a/scripts/run_fireworks_suite.sh +++ b/scripts/run_fireworks_suite.sh @@ -67,9 +67,11 @@ run_step() { # Measured on inkling-small via OpenRouter: effort=high returns content=None # after ~233s per sample (every sample would burn 3 retries and fail), while # effort=none answers in ~1.3s. Transcription gains nothing from thinking. -BENCH_OPENROUTER_EFFORT="$ASR_EFFORT" \ -run_step asr uv run -m benchmarks.asr.voxpopuli_aa_multi \ - --provider "$PROVIDER" --model "$MODEL" +if [ -n "${TARGET:-}" ]; then + run_step asr uv run python -m bench_core run --benchmark asr --target "$TARGET" +else + echo " [skip] asr: set TARGET= (python -m bench_core list-targets)" +fi run_step spider2 uv run -m benchmarks.spider2_lite.spider2_lite \ --provider "$PROVIDER" --model "$MODEL" diff --git a/tests/test_asr_bench.py b/tests/test_asr_bench.py new file mode 100644 index 0000000..40d5ffe --- /dev/null +++ b/tests/test_asr_bench.py @@ -0,0 +1,76 @@ +"""ASR benchmark: normalize/parse units + offline scoring-parity replay. + +The parity test replays each archived ASR run through the new score and asserts +corpus WER, time-weighted WER, CER, and counts match the archived metrics — +proving the port reproduces the numbers with no audio calls. +""" + +import json +import pathlib +from types import SimpleNamespace as NS + +import pytest + +from benchmarks.asr import bench + +RESULTS = pathlib.Path(__file__).resolve().parent.parent / "results" +ARCHIVED_TAGS = [ + "voxpopuli_aa_fireworks_accounts_fireworks_models_inkling", + "voxpopuli_aa_gemini_gemini-3.7-flash", + "voxpopuli_aa_openrouter_thinkingmachines_inkling-small", + "voxpopuli_aa_openrouter_google_gemini-3.7-flash", +] + + +def test_normalize_text_whisper_style(): + assert bench.normalize_text("Hello, WORLD!") == "hello world" + assert bench.normalize_text("it's fine\n") == "it's fine" + assert bench.normalize_text("") == "" + + +def test_parse_strips_transcript(): + assert bench.parse(NS(text=" hi there \n"), None) == "hi there" + + +def test_score_corpus_wer_basic(): + samples = [ + {"id": "1", "transcript": "the cat sat", "duration": 1.0}, + {"id": "2", "transcript": "hello world", "duration": 1.0}, + ] + records = [ + {"id": "1", "prediction": "the cat sat"}, # 0 errors + {"id": "2", "prediction": "hello there"}, # 1/2 words wrong + ] + m = bench.score(records, samples) + assert m["num_samples"] == 2 + assert m["corpus_wer"] == pytest.approx(1 / 5) # 1 error over 5 ref words + + +@pytest.mark.parametrize("tag", ARCHIVED_TAGS) +def test_scoring_parity_with_archived_run(tag): + resp_path = RESULTS / f"{tag}_responses.jsonl" + metrics_path = RESULTS / f"{tag}_metrics.json" + if not (resp_path.exists() and metrics_path.exists()): + pytest.skip(f"archived run {tag} not present") + + archived = json.loads(metrics_path.read_text()) + samples, records = [], [] + for line in resp_path.read_text().splitlines(): + if not line.strip(): + continue + rec = json.loads(line) + samples.append( + { + "id": rec["id"], + "transcript": rec["transcript"], + "duration": rec.get("duration"), + } + ) + records.append({"id": rec["id"], "prediction": rec["response"]}) + + m = bench.score(records, samples) + assert m["num_samples"] == archived["num_samples"] + assert m["corpus_wer"] == pytest.approx(archived["corpus_wer"]) + assert m["corpus_cer"] == pytest.approx(archived["corpus_cer"]) + assert m["time_weighted_wer"] == pytest.approx(archived["time_weighted_wer"]) + assert m["mean_sample_wer"] == pytest.approx(archived["mean_sample_wer"]) From 2742539d04ceb87d66087ab4fa48be0db58f4d52 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 08:40:59 +0530 Subject: [PATCH 09/38] feat(bench_core): migrate MMMLU onto the abstraction; retire 2 forks benchmarks/mmmlu/bench.py (14-language lite MCQ, per-language macro accuracy) replaces mmmlu.py + mmmlu_multi.py. Offline parity replays the 4 archived runs (inkling macro 0.8846, gemini 0.9268, deepseek 0.7961, inkling-small 0.8737). CLI --benchmark mmmlu; suite + README updated. --- README.md | 18 +- bench_core/cli.py | 1 + benchmarks/mmmlu/bench.py | 148 +++++++ benchmarks/mmmlu/mmmlu.py | 472 --------------------- benchmarks/mmmlu/mmmlu_multi.py | 718 -------------------------------- scripts/run_fireworks_suite.sh | 8 +- tests/test_mmmlu_bench.py | 67 +++ 7 files changed, 227 insertions(+), 1205 deletions(-) create mode 100644 benchmarks/mmmlu/bench.py delete mode 100644 benchmarks/mmmlu/mmmlu.py delete mode 100644 benchmarks/mmmlu/mmmlu_multi.py create mode 100644 tests/test_mmmlu_bench.py diff --git a/README.md b/README.md index 58a1726..899aa66 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ audio input, so it runs on every benchmark here: uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks uv run -m benchmarks.olmocr.olmocr_bench_fireworks uv run python -m bench_core run --target inkling --benchmark gpqa -uv run -m benchmarks.mmmlu.mmmlu_multi --provider fireworks --model accounts/fireworks/models/inkling +uv run python -m bench_core run --target inkling --benchmark mmmlu uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider fireworks --model accounts/fireworks/models/inkling --setting standard uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider fireworks --model accounts/fireworks/models/inkling --setting vision uv run -m benchmarks.obj_detection.refcoco_multi --provider fireworks --model accounts/fireworks/models/inkling @@ -231,18 +231,10 @@ Links: [dataset](https://huggingface.co/datasets/openai/MMMLU) 14 languages, exact-match accuracy macro-averaged across languages. ```bash -# Interfaze -uv run -m benchmarks.mmmlu.mmmlu -uv run -m benchmarks.mmmlu.mmmlu --languages DE_DE FR_FR # subset of languages - -# Any provider -uv run -m benchmarks.mmmlu.mmmlu_multi --provider openai --model gpt-5.4-mini -uv run -m benchmarks.mmmlu.mmmlu_multi --provider gemini --model gemini-3.1-pro-preview -uv run -m benchmarks.mmmlu.mmmlu_multi --provider anthropic --model claude-sonnet-4-6 -uv run -m benchmarks.mmmlu.mmmlu_multi --provider interfaze --model interfaze-beta - -# Evaluate only -uv run -m benchmarks.mmmlu.mmmlu --evaluate-only +# Any target (MMMLU-lite, 14 languages, macro-averaged) +uv run python -m bench_core run --target gpt-5.5 --benchmark mmmlu +uv run python -m bench_core run --target gemini-3.7-flash --benchmark mmmlu --reasoning high +uv run python -m bench_core run --target inkling --benchmark mmmlu ``` --- diff --git a/bench_core/cli.py b/bench_core/cli.py index 2b080d5..04ca252 100644 --- a/bench_core/cli.py +++ b/bench_core/cli.py @@ -19,6 +19,7 @@ BENCHMARKS = { "gpqa": "benchmarks.gpqa.bench", "asr": "benchmarks.asr.bench", + "mmmlu": "benchmarks.mmmlu.bench", } diff --git a/benchmarks/mmmlu/bench.py b/benchmarks/mmmlu/bench.py new file mode 100644 index 0000000..db94369 --- /dev/null +++ b/benchmarks/mmmlu/bench.py @@ -0,0 +1,148 @@ +"""MMMLU-lite: 14-language MCQ; per-language accuracy, macro-averaged headline.""" + +from __future__ import annotations + +import re +from collections import defaultdict + +from bench_core.request import Message, ReasoningSpec, Request, TextPart + +NAME = "mmmlu" +ID_KEY = "id" +PRIMARY_METRIC = "macro_accuracy" +DEFAULTS = {"reasoning": "off", "rate_limit": 50, "max_in_flight": 8} + +_DATASET_LITE_ID = "opencompass/mmmlu_lite" +_SPLIT = "test" +LANGUAGES = [ + "AR_XY", + "BN_BD", + "DE_DE", + "ES_LA", + "FR_FR", + "HI_IN", + "ID_ID", + "IT_IT", + "JA_JP", + "KO_KR", + "PT_BR", + "SW_KE", + "YO_NG", + "ZH_CN", +] + +# English meta-instruction held constant across all languages so the parser can +# rely on Latin A-D output. +_PROMPT = ( + "The following is a multiple choice question. Respond with only a single " + "letter (A, B, C, or D) corresponding to the correct answer. Do not " + "explain your reasoning.\n\n" + "Question: {question}\n" + "A. {a}\nB. {b}\nC. {c}\nD. {d}\n\n" + "Answer:" +) + +_LETTER_RE = re.compile(r"\b([ABCD])\b") +_FIRST_LETTER_RE = re.compile(r"[ABCD]") + + +def parse_answer(text: str) -> str | None: + if not text: + return None + s = text.strip() + if len(s) == 1 and s.upper() in "ABCD": + return s.upper() + m = _LETTER_RE.search(s.upper()) + if m: + return m.group(1) + m = _FIRST_LETTER_RE.search(s.upper()) + return m.group(0) if m else None + + +def load_samples(sample_size: int | None = None) -> list[dict]: + from datasets import load_dataset + + samples = [] + for lang in LANGUAGES: + ds = load_dataset(_DATASET_LITE_ID, lang, split=_SPLIT) + for i, row in enumerate(ds): + row = dict(row) + samples.append( + { + "id": f"{lang}:{i}", + "language": lang, + "subject": row["subject"], + "question": row["input"], + "a": row["A"], + "b": row["B"], + "c": row["C"], + "d": row["D"], + "answer": str(row["target"]).strip().upper(), + } + ) + return samples[:sample_size] if sample_size else samples + + +def build_request(sample: dict, mode: str) -> Request: + prompt = _PROMPT.format( + question=sample["question"], + a=sample["a"], + b=sample["b"], + c=sample["c"], + d=sample["d"], + ) + return Request( + messages=[Message("user", [TextPart(prompt)])], + reasoning=ReasoningSpec(mode), + temperature=0.0, + ) + + +def parse(response, sample) -> str | None: + return parse_answer(response.text or "") + + +def score(records: list[dict], samples: list[dict]) -> dict: + by_id = {s["id"]: s for s in samples} + # de-dupe records by id (last-wins), matching the old load_records + latest = {r["id"]: r for r in records if r["id"] in by_id} + + by_lang: dict[str, list[bool]] = defaultdict(list) + by_subject: dict[str, list[bool]] = defaultdict(list) + unparse: dict[str, int] = defaultdict(int) + for rid, r in latest.items(): + s = by_id[rid] + pred = r.get("prediction") + correct = pred == s["answer"] + by_lang[s["language"]].append(correct) + by_subject[s["subject"]].append(correct) + if pred is None: + unparse[s["language"]] += 1 + + per_language = { + lang: { + "n": len(v), + "accuracy": sum(v) / len(v) if v else 0.0, + "unparseable": unparse[lang], + } + for lang, v in by_lang.items() + } + per_subject = { + subj: {"n": len(v), "accuracy": sum(v) / len(v) if v else 0.0} + for subj, v in by_subject.items() + } + + lang_accs = [v["accuracy"] for v in per_language.values()] + macro = sum(lang_accs) / len(lang_accs) if lang_accs else 0.0 + n_total = sum(v["n"] for v in per_language.values()) + # micro reconstructed the same lossy way the original did (float->int per lang) + n_correct = sum(int(v["accuracy"] * v["n"]) for v in per_language.values()) + micro = n_correct / n_total if n_total else 0.0 + + return { + "macro_accuracy": macro, + "micro_accuracy": micro, + "num_samples": n_total, + "per_language": per_language, + "per_subject": per_subject, + } diff --git a/benchmarks/mmmlu/mmmlu.py b/benchmarks/mmmlu/mmmlu.py deleted file mode 100644 index af5e035..0000000 --- a/benchmarks/mmmlu/mmmlu.py +++ /dev/null @@ -1,472 +0,0 @@ -""" -MMMLU (Multilingual MMLU) benchmark for Interfaze. - -OpenAI's professional-translation version of the MMLU test set across 14 -languages — the same benchmark Gemini 3 Pro reports as "Multilingual Q&A" -(91.8% in their Nov 2025 model card). Each language subset is the full MMLU -test split (~14,042 four-choice questions across 57 subjects) translated by -human translators. - -Dataset: https://huggingface.co/datasets/openai/MMMLU - -Languages (14): - AR_XY Arabic - BN_BD Bengali - DE_DE German - ES_LA Spanish (Latin America) - FR_FR French - HI_IN Hindi - ID_ID Indonesian - IT_IT Italian - JA_JP Japanese - KO_KR Korean - PT_BR Brazilian Portuguese - SW_KE Swahili - YO_NG Yoruba - ZH_CN Simplified Chinese - -Methodology (matches Gemini 3 Pro model card): - pass@1 — single attempt, no majority voting, no parallel test-time compute. - Default sampling, single trial (large benchmark, no trial averaging). - Headline number = macro-average accuracy across the 14 languages. - -Metric: exact-match accuracy on the predicted answer letter (A/B/C/D). - Reported per-language, per-subject, and macro-averaged. - -Usage: - uv run -m benchmarks.mmmlu.mmmlu # full run, all 14 langs - uv run -m benchmarks.mmmlu.mmmlu --limit 50 # smoke test: 50 per lang - uv run -m benchmarks.mmmlu.mmmlu --languages DE_DE FR_FR # subset of langs - uv run -m benchmarks.mmmlu.mmmlu --evaluate-only # rescore existing preds -""" - -import sys -import os -import re -import json -import time -import asyncio -import argparse -import traceback -from pathlib import Path -from collections import defaultdict - -from datasets import load_dataset -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -from src.commons import invoke_interfaze # noqa: E402 - -RESULTS_DIR = PROJECT_ROOT / "results" -DATASET_ID = "openai/MMMLU" -SPLIT = "test" - -LANGUAGES = [ - "AR_XY", "BN_BD", "DE_DE", "ES_LA", "FR_FR", "HI_IN", "ID_ID", - "IT_IT", "JA_JP", "KO_KR", "PT_BR", "SW_KE", "YO_NG", "ZH_CN", -] - -REASONING_EFFORT = None # off — Gemini 3 Pro reports MMMLU as a non-thinking eval -TEMPERATURE = 0.0 # deterministic; repo convention (Gemini uses default sampling) -RATE_LIMIT = 25 -MAX_RETRIES = 3 - -# English instruction is intentional and standard for MMMLU evaluation: the -# question + options are in the target language, but the meta-instruction -# ("answer with a single letter") is held constant so the parser can rely on -# Latin A-D output regardless of the language of the question. -PROMPT_TEMPLATE = ( - "The following is a multiple choice question. Respond with only a single " - "letter (A, B, C, or D) corresponding to the correct answer. Do not " - "explain your reasoning.\n\n" - "Question: {question}\n" - "A. {a}\n" - "B. {b}\n" - "C. {c}\n" - "D. {d}\n\n" - "Answer:" -) - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -class JsonlWriter: - def __init__(self, path: Path): - self.path = path - self.path.parent.mkdir(parents=True, exist_ok=True) - self._lock = asyncio.Lock() - - async def append(self, record: dict): - line = json.dumps(record, ensure_ascii=False) - async with self._lock: - with open(self.path, "a", encoding="utf-8") as f: - f.write(line + "\n") - f.flush() - os.fsync(f.fileno()) - - -def load_completed_ids(path: Path) -> set[str]: - if not path.exists(): - return set() - done: set[str] = set() - with open(path, encoding="utf-8") as f: - for line_no, line in enumerate(f, 1): - line = line.strip() - if not line: - continue - try: - rec = json.loads(line) - except json.JSONDecodeError: - tqdm.write(f"[resume] skipping malformed line {line_no} in {path}") - continue - if rec.get("response") is not None: - done.add(str(rec["id"])) - return done - - -def load_records(path: Path) -> list[dict]: - records: list[dict] = [] - if not path.exists(): - return records - with open(path, encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - records.append(json.loads(line)) - except json.JSONDecodeError: - continue - by_id: dict[str, dict] = {} - for r in records: - by_id[str(r["id"])] = r - return list(by_id.values()) - - -_LETTER_RE = re.compile(r"\b([ABCD])\b") -_FIRST_LETTER_RE = re.compile(r"[ABCD]") - - -def parse_answer(text: str) -> str | None: - """Extract a single A/B/C/D letter from the model's response. - - Try a word-boundary match first (handles "A", "A.", "(A)", "Answer: A"). - Fall back to the first standalone-looking letter; return None if nothing matches.""" - if not text: - return None - stripped = text.strip() - # Direct hit: response is just the letter. - if len(stripped) == 1 and stripped.upper() in "ABCD": - return stripped.upper() - m = _LETTER_RE.search(stripped.upper()) - if m: - return m.group(1) - m = _FIRST_LETTER_RE.search(stripped.upper()) - if m: - return m.group(0) - return None - - -def build_sample(row: dict, language: str) -> dict: - idx = row.get("Unnamed: 0") - return { - "id": f"{language}:{idx}", - "language": language, - "row_index": idx, - "subject": row["Subject"], - "question": row["Question"], - "a": row["A"], - "b": row["B"], - "c": row["C"], - "d": row["D"], - "answer": str(row["Answer"]).strip().upper(), - } - - -async def process_sample(sample: dict, rate_limiter: RateLimiter, - writer: JsonlWriter, progress: dict) -> dict | None: - last_error: str | None = None - - prompt = PROMPT_TEMPLATE.format( - question=sample["question"], - a=sample["a"], b=sample["b"], c=sample["c"], d=sample["d"], - ) - messages = [{"role": "user", "content": prompt}] - - for attempt in range(1, MAX_RETRIES + 1): - await rate_limiter.acquire() - start = time.perf_counter() - try: - response = await asyncio.to_thread( - invoke_interfaze, - messages, - reasoning_effort=REASONING_EFFORT, - temperature=TEMPERATURE, - ) - latency_ms = int((time.perf_counter() - start) * 1000) - content = (response.choices[0].message.content or "").strip() - request_id = getattr(response, "id", None) - if not content: - last_error = "empty response content" - raise RuntimeError(last_error) - - predicted = parse_answer(content) - correct = predicted == sample["answer"] - - record = { - "id": sample["id"], - "language": sample["language"], - "row_index": sample["row_index"], - "subject": sample["subject"], - "answer": sample["answer"], - "prediction": predicted, - "correct": correct, - "response": content, - "request_id": request_id, - "latency_ms": latency_ms, - "attempts": attempt, - } - await writer.append(record) - - progress["done"] += 1 - if correct: - progress["correct"] += 1 - if predicted is None: - progress["unparseable"] += 1 - tqdm.write( - f"[{progress['done']}/{progress['total']}] " - f"{sample['language']} subj={sample['subject'][:18]:18} " - f"gold={sample['answer']} pred={predicted or '?'} " - f"{'OK' if correct else 'X '} latency={latency_ms}ms" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - tqdm.write( - f"[error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(2 ** (attempt - 1)) - - progress["failed"] += 1 - tqdm.write(f"[FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") - return None - - -def compute_metrics(results: list[dict]) -> dict: - if not results: - return {} - - by_lang: dict[str, list[dict]] = defaultdict(list) - by_subject: dict[str, list[dict]] = defaultdict(list) - for r in results: - by_lang[r["language"]].append(r) - by_subject[r["subject"]].append(r) - - per_language: dict[str, dict] = {} - for lang, rows in by_lang.items(): - n = len(rows) - n_correct = sum(1 for r in rows if r.get("correct")) - n_unparseable = sum(1 for r in rows if r.get("prediction") is None) - per_language[lang] = { - "n": n, - "accuracy": n_correct / n if n else 0.0, - "unparseable": n_unparseable, - } - - per_subject: dict[str, dict] = {} - for subj, rows in by_subject.items(): - n = len(rows) - n_correct = sum(1 for r in rows if r.get("correct")) - per_subject[subj] = { - "n": n, - "accuracy": n_correct / n if n else 0.0, - } - - # Macro-average across languages — this is the headline MMMLU number. - lang_accs = [v["accuracy"] for v in per_language.values()] - macro_accuracy = sum(lang_accs) / len(lang_accs) if lang_accs else 0.0 - - # Micro-accuracy = pooled across all samples (depends on per-lang counts). - n_total = sum(v["n"] for v in per_language.values()) - n_total_correct = sum(int(v["accuracy"] * v["n"]) for v in per_language.values()) - micro_accuracy = n_total_correct / n_total if n_total else 0.0 - - latencies = [r["latency_ms"] for r in results if isinstance(r.get("latency_ms"), int)] - latency_stats = {} - if latencies: - lats = sorted(latencies) - n = len(lats) - latency_stats = { - "count": n, - "mean_ms": sum(lats) / n, - "p50_ms": lats[n // 2], - "p90_ms": lats[min(n - 1, int(n * 0.9))], - "p99_ms": lats[min(n - 1, int(n * 0.99))], - "max_ms": lats[-1], - } - - return { - "macro_accuracy": macro_accuracy, - "micro_accuracy": micro_accuracy, - "num_samples": n_total, - "per_language": per_language, - "per_subject": per_subject, - "latency": latency_stats, - } - - -def print_summary(metrics: dict): - print(f"\n{'=' * 68}") - print(f"MMMLU Results (Interfaze, reasoning={REASONING_EFFORT}, temp={TEMPERATURE})") - print(f"{'=' * 68}") - print(f"Samples : {metrics['num_samples']}") - print(f"Macro-avg accuracy : {metrics['macro_accuracy']:.4f} ← headline (Gemini 3 Pro: 0.918)") - print(f"Micro-avg accuracy : {metrics['micro_accuracy']:.4f}") - print() - print("Per-language accuracy:") - print(f" {'lang':6} {'n':>6} {'acc':>8} {'unparseable':>12}") - for lang in sorted(metrics["per_language"].keys()): - v = metrics["per_language"][lang] - print(f" {lang:6} {v['n']:>6} {v['accuracy']:>8.4f} {v['unparseable']:>12}") - if metrics.get("latency"): - lat = metrics["latency"] - print(f"\nLatency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " - f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms max={lat['max_ms']}ms") - - -def load_all_samples(languages: list[str], limit: int | None) -> list[dict]: - """Load samples from each language subset, optionally capping per language.""" - all_samples: list[dict] = [] - for lang in languages: - print(f"Loading {DATASET_ID}/{lang} (split={SPLIT})...") - ds = load_dataset(DATASET_ID, lang, split=SPLIT) - rows = [build_sample(dict(row), lang) for row in ds] - if limit is not None: - rows = rows[:limit] - print(f" -> {len(rows)} samples") - all_samples.extend(rows) - return all_samples - - -async def run_predictions(pred_path: Path, languages: list[str], limit: int | None): - samples = load_all_samples(languages, limit) - done_ids = load_completed_ids(pred_path) - pending = [s for s in samples if s["id"] not in done_ids] - print(f"Total samples: {len(samples)}") - print(f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})") - if not pending: - return - - writer = JsonlWriter(pred_path) - rate_limiter = RateLimiter(RATE_LIMIT) - progress = { - "total": len(pending), - "done": 0, - "correct": 0, - "unparseable": 0, - "failed": 0, - } - - tasks = [process_sample(s, rate_limiter, writer, progress) for s in pending] - try: - await tqdm_asyncio.gather(*tasks, desc="MMMLU") - except Exception: - traceback.print_exc() - - acc = progress["correct"] / progress["done"] if progress["done"] else 0.0 - print( - f"\nRun finished: {progress['done']}/{progress['total']} answered " - f"({progress['failed']} failed, {progress['unparseable']} unparseable). " - f"Pooled accuracy on this run: {acc:.4f}" - ) - - -def run_evaluation(pred_path: Path, metrics_path: Path): - if not pred_path.exists(): - print(f"No predictions found at {pred_path}") - sys.exit(1) - results = load_records(pred_path) - if not results: - print(f"No records in {pred_path}") - sys.exit(1) - - # Re-derive `correct` and `prediction` if missing (e.g., re-scoring an old run). - for r in results: - if r.get("prediction") is None and r.get("response"): - r["prediction"] = parse_answer(r["response"]) - if r.get("correct") is None and r.get("prediction") is not None: - r["correct"] = r["prediction"] == r.get("answer") - - metrics = compute_metrics(results) - print_summary(metrics) - output = { - **metrics, - "dataset": DATASET_ID, - "split": SPLIT, - "languages": sorted({r["language"] for r in results}), - "reasoning_effort": REASONING_EFFORT, - "temperature": TEMPERATURE, - "rate_limit": RATE_LIMIT, - "model": "interfaze-beta", - } - metrics_path.parent.mkdir(parents=True, exist_ok=True) - with open(metrics_path, "w") as f: - json.dump(output, f, indent=2, ensure_ascii=False) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - parser = argparse.ArgumentParser(description="MMMLU benchmark for Interfaze") - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument( - "--languages", nargs="+", default=LANGUAGES, - choices=LANGUAGES, - help="Subset of languages to run (default: all 14).", - ) - parser.add_argument( - "--limit", type=int, default=None, - help="Cap samples per language (smoke test).", - ) - args = parser.parse_args() - - pred_path = RESULTS_DIR / "mmmlu_responses.jsonl" - metrics_path = RESULTS_DIR / "mmmlu_metrics.json" - - if args.evaluate_only: - run_evaluation(pred_path, metrics_path) - elif args.predict_only: - asyncio.run(run_predictions(pred_path, args.languages, args.limit)) - else: - asyncio.run(run_predictions(pred_path, args.languages, args.limit)) - run_evaluation(pred_path, metrics_path) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/mmmlu/mmmlu_multi.py b/benchmarks/mmmlu/mmmlu_multi.py deleted file mode 100644 index 71877ca..0000000 --- a/benchmarks/mmmlu/mmmlu_multi.py +++ /dev/null @@ -1,718 +0,0 @@ -""" -MMMLU benchmark — multi-provider runner. - -Same dataset, prompt, parser, and scoring as benchmarks.mmmlu.mmmlu (the -interfaze run), but routed through other providers (OpenAI, Anthropic, Google) -for head-to-head comparison. - -Methodology mirrors Gemini 3 Pro's published MMMLU setup: - - pass@1, single trial, no majority voting - - reasoning OFF where the model supports it (lowest available level) - - temperature=0 where the API allows it (some providers reject t=0 + thinking) - - macro-average accuracy across the 14 translated languages = headline number - -Output (per provider+model, so parallel runs don't clash): - results/mmmlu___responses.jsonl - results/mmmlu___metrics.json - -Usage: - uv run -m benchmarks.mmmlu.mmmlu_multi --provider interfaze --model interfaze-beta - uv run -m benchmarks.mmmlu.mmmlu_multi --provider openai --model gpt-5.4-mini - uv run -m benchmarks.mmmlu.mmmlu_multi --provider gemini --model gemini-3.1-pro-preview - uv run -m benchmarks.mmmlu.mmmlu_multi --provider anthropic --model claude-sonnet-4-6 - uv run -m benchmarks.mmmlu.mmmlu_multi --provider fireworks --model accounts/fireworks/models/inkling - # Smoke (1 sample per language = 14 total): - uv run -m benchmarks.mmmlu.mmmlu_multi --provider gemini --model gemini-3-flash-preview --limit 1 - -Env: OPENAI_API_KEY, GEMINI_KEY, ANTHROPIC_API_KEY, INTERFAZE_API_KEY (loaded from .env). -""" - -import os -import re -import sys -import json -import time -import asyncio -import argparse -import traceback -from pathlib import Path - -from datasets import load_dataset -from dotenv import load_dotenv -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -# Reuse helpers from the interfaze base script — guarantees identical prompt / -# parser / scoring across providers. -from benchmarks.mmmlu.mmmlu import ( # noqa: E402 - DATASET_ID, - SPLIT, - LANGUAGES, - PROMPT_TEMPLATE, - JsonlWriter, - RateLimiter, - build_sample, - compute_metrics, - load_completed_ids, - load_records, - parse_answer, - print_summary as _base_print_summary, -) - -load_dotenv() - - -def _load_interfaze_env_fallback() -> None: - """Pull any keys from ~/interfaze/.env.local that aren't already in os.environ. - Lets us run providers whose keys live there (e.g. ANTHROPIC_API_KEY) without - requiring users to duplicate them into the project .env.""" - path = Path.home() / "interfaze" / ".env.local" - if not path.exists(): - return - for line in path.read_text().splitlines(): - line = line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - k, v = line.split("=", 1) - k = k.strip() - v = v.strip().strip('"').strip("'") - os.environ.setdefault(k, v) - - -_load_interfaze_env_fallback() - -RESULTS_DIR = PROJECT_ROOT / "results" -RATE_LIMIT = 50 -MAX_RETRIES = 3 - -# Cap on requests in flight. RATE_LIMIT bounds how many requests *start* per -# second but not how many are outstanding: gather() launches every pending -# sample at once, so hundreds pile up, queue server-side, and eventually 429 -# after tens of minutes of latency. Override with BENCH_MAX_IN_FLIGHT. -MAX_IN_FLIGHT = int(os.getenv("BENCH_MAX_IN_FLIGHT", "8")) - -def _gemini_floor(model: str) -> str: - """Lowest thinking_level the given Gemini model actually accepts. - - gemini-3.7+ flash rejects MINIMAL server-side ("Thinking level MINIMAL is - not supported for this model"), and the SDK enum has no "off"/"none", so - "low" is its true floor. Older 3.x flash models still take "minimal". - BENCH_GEMINI_FLOOR overrides for anything not covered here. - """ - override = os.getenv("BENCH_GEMINI_FLOOR") - if override: - return override - m = model.lower() - if any(v in m for v in ("3.7", "3.8", "3.9", "4.")): - return "low" - return "minimal" -DEFAULT_TEMPERATURE = 0.0 - -# Dataset variant — "full" (openai/MMMLU, ~196k) or "lite" (opencompass/mmmlu_lite, -# ~20k stratified). Lite uses a different schema (input/target vs Question/Answer). -DATASET_VARIANT = "lite" - -DATASET_FULL_ID = "openai/MMMLU" -DATASET_LITE_ID = "opencompass/mmmlu_lite" - -# Reasoning mode — "off" (each model at its floor) or "high" (each model at -# max). Mutated by CLI before any inference runs. -REASONING_MODE = "off" - -# Anthropic thinking budget (only used when REASONING_MODE == "high"). max_tokens -# must be > budget_tokens; we set 16k cap with 10k thinking budget. -ANTHROPIC_HIGH_BUDGET_TOKENS = 10_000 -ANTHROPIC_HIGH_MAX_TOKENS = 16_000 -ANTHROPIC_OFF_MAX_TOKENS = 512 - - -# --------------------------------------------------------------------------- -# Dataset loading + sample building -# --------------------------------------------------------------------------- - - -def build_sample_lite(row: dict, language: str, row_index: int) -> dict: - """opencompass/mmmlu_lite uses input/target/A-D/subject (no Unnamed: 0).""" - return { - "id": f"{language}:{row_index}", - "language": language, - "row_index": row_index, - "subject": row["subject"], - "question": row["input"], - "a": row["A"], - "b": row["B"], - "c": row["C"], - "d": row["D"], - "answer": str(row["target"]).strip().upper(), - } - - -def load_dataset_for_variant(variant: str, lang: str): - if variant == "lite": - return load_dataset(DATASET_LITE_ID, lang, split="test") - return load_dataset(DATASET_FULL_ID, lang, split="test") - - -def build_sample_for_variant( - variant: str, row: dict, language: str, row_index: int -) -> dict: - if variant == "lite": - return build_sample_lite(row, language, row_index) - return build_sample(row, language) - - -# --------------------------------------------------------------------------- -# Provider adapters: each takes (prompt, model, client) and returns -# (content: str, request_id: str | None, usage: dict) -# `usage` keys: input_tokens, output_tokens, reasoning_tokens (or None if -# the provider doesn't report it). Lets us verify whether reasoning is -# actually OFF rather than just configured off. -# Run synchronously inside asyncio.to_thread. -# --------------------------------------------------------------------------- - - -def _safe_int(x): - try: - return int(x) if x is not None else None - except (TypeError, ValueError): - return None - - -def _openai_usage(resp) -> dict: - """Extract input/output/reasoning tokens from an OpenAI chat.completions response.""" - u = getattr(resp, "usage", None) - if u is None: - return {"input_tokens": None, "output_tokens": None, "reasoning_tokens": None} - details = getattr(u, "completion_tokens_details", None) - reasoning = getattr(details, "reasoning_tokens", None) if details else None - if reasoning is None: - # Fireworks reports it at the top level of `usage` for Inkling. - reasoning = getattr(u, "reasoning_tokens", None) - return { - "input_tokens": _safe_int(getattr(u, "prompt_tokens", None)), - "output_tokens": _safe_int(getattr(u, "completion_tokens", None)), - "reasoning_tokens": _safe_int(reasoning), - } - - -def _gemini_usage(resp) -> dict: - """Extract usage from a google-genai response. Gemini reports thinking - tokens as `thoughts_token_count` — 0 confirms reasoning fully off.""" - u = getattr(resp, "usage_metadata", None) - if u is None: - return {"input_tokens": None, "output_tokens": None, "reasoning_tokens": None} - return { - "input_tokens": _safe_int(getattr(u, "prompt_token_count", None)), - "output_tokens": _safe_int(getattr(u, "candidates_token_count", None)), - "reasoning_tokens": _safe_int(getattr(u, "thoughts_token_count", None)), - } - - -def _anthropic_usage(resp) -> dict: - """Anthropic — `output_tokens` includes thinking tokens when extended - thinking is on. With thinking disabled, no thinking content blocks - appear, so output_tokens is purely the visible answer.""" - u = getattr(resp, "usage", None) - if u is None: - return {"input_tokens": None, "output_tokens": None, "reasoning_tokens": None} - return { - "input_tokens": _safe_int(getattr(u, "input_tokens", None)), - "output_tokens": _safe_int(getattr(u, "output_tokens", None)), - "reasoning_tokens": None, # not separately reported - } - - -def call_interfaze(prompt: str, model: str, client) -> tuple[str, str | None, dict]: - """Interfaze uses 'off'|'high' for reasoning_effort (NOT 'none'; that's - OpenAI's vocabulary). Valid values per the API: minimal|low|medium|high|on|off|auto.""" - kwargs = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "temperature": DEFAULT_TEMPERATURE, - "reasoning_effort": "off" if REASONING_MODE == "off" else "high", - } - resp = client.chat.completions.create(**kwargs) - return ( - (resp.choices[0].message.content or "").strip(), - getattr(resp, "id", None), - _openai_usage(resp), - ) - - -def call_openai(prompt: str, model: str, client) -> tuple[str, str | None, dict]: - """GPT-5.x — reasoning off vs high. With reasoning engaged the API rejects - temperature!=default, so we omit temperature in 'high' mode.""" - kwargs = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - } - if REASONING_MODE == "off": - kwargs["reasoning_effort"] = "none" - kwargs["temperature"] = DEFAULT_TEMPERATURE - else: - kwargs["reasoning_effort"] = "high" - resp = client.chat.completions.create(**kwargs) - return ( - (resp.choices[0].message.content or "").strip(), - getattr(resp, "id", None), - _openai_usage(resp), - ) - - -def call_anthropic(prompt: str, model: str, client) -> tuple[str, str | None, dict]: - """Claude — thinking explicitly disabled vs enabled with a high budget. - With thinking enabled, Anthropic requires temperature=1, so we omit it.""" - kwargs = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - } - if REASONING_MODE == "off": - kwargs["thinking"] = {"type": "disabled"} - kwargs["temperature"] = DEFAULT_TEMPERATURE - kwargs["max_tokens"] = ANTHROPIC_OFF_MAX_TOKENS - else: - kwargs["thinking"] = { - "type": "enabled", - "budget_tokens": ANTHROPIC_HIGH_BUDGET_TOKENS, - } - kwargs["max_tokens"] = ANTHROPIC_HIGH_MAX_TOKENS - resp = client.messages.create(**kwargs) - parts = [b.text for b in resp.content if getattr(b, "type", None) == "text"] - return "\n".join(parts).strip(), resp.id, _anthropic_usage(resp) - - -def _openrouter_extra_body(model: str) -> dict: - """Per-model OpenRouter extras. Models differ on what `reasoning` shapes - they accept and which underlying provider should serve them. - - - x-ai/grok-4.3: rejects `enabled=false`, so `off` maps to the lowest - accepted tier (`effort=minimal`); `high` = default thinking on. - - moonshotai/kimi-k2.6: supports `enabled=false`; pin provider to Moonshot - so we benchmark Moonshot's own deployment, not a downstream reseller. - - default: assume the model accepts the unified `enabled` toggle. - """ - m = model.lower() - if m.startswith("x-ai/grok-4.3"): - if REASONING_MODE == "off": - return {"reasoning": {"effort": "minimal"}} - return {"reasoning": {"enabled": True}} - if m.startswith("google/"): - # Gemini rejects reasoning.enabled=false outright ("Reasoning is - # mandatory for this endpoint and cannot be disabled"), so "off" maps to - # its real floor: effort=minimal, measured at 0 reasoning tokens. This - # matches what call_gemini does natively (thinking_level="minimal"). - return {"reasoning": {"effort": "minimal" if REASONING_MODE == "off" else "high"}} - if m.startswith("moonshotai/"): - body = {"reasoning": {"enabled": REASONING_MODE != "off"}} - body["provider"] = {"only": ["moonshotai"]} - return body - return {"reasoning": {"enabled": REASONING_MODE != "off"}} - - -def call_openrouter(prompt: str, model: str, client) -> tuple[str, str | None, dict]: - """OpenRouter dispatch — see _openrouter_extra_body for per-model knobs. - temperature=0 is accepted across the models we currently route here.""" - resp = client.chat.completions.create( - model=model, - messages=[{"role": "user", "content": prompt}], - temperature=DEFAULT_TEMPERATURE, - extra_body=_openrouter_extra_body(model), - ) - return ( - (resp.choices[0].message.content or "").strip(), - getattr(resp, "id", None), - _openai_usage(resp), - ) - - -def call_fireworks(prompt: str, model: str, client) -> tuple[str, str | None, dict]: - """Fireworks-hosted models (Inkling). Inkling always thinks — reasoning_effort - accepts none|low|medium|high|xhigh|max and "none" is the floor, not a true - off, so `reasoningoff` runs here still emit reasoning tokens (recorded per - sample in the output JSONL).""" - resp = client.chat.completions.create( - model=model, - messages=[{"role": "user", "content": prompt}], - temperature=DEFAULT_TEMPERATURE, - reasoning_effort="none" if REASONING_MODE == "off" else "high", - ) - return ( - (resp.choices[0].message.content or "").strip(), - getattr(resp, "id", None), - _openai_usage(resp), - ) - - -def call_gemini(prompt: str, model: str, client) -> tuple[str, str | None, dict]: - """Gemini — off goes to each model's floor; high goes to max thinking. - 3.x Pro: 'low' floor / 'high' max. - 3.x Flash: 'minimal' floor / 'high' max. - 2.5 Pro: budget=128 floor / budget=-1 (dynamic, model-decides) for high. - 2.5 Flash: budget=0 floor / budget=-1 for high. - Temperature 0 is fine with thinking on for Gemini.""" - from google.genai import types - - m = model.lower() - if REASONING_MODE == "off": - if m.startswith("gemini-2.5-pro"): - thinking = types.ThinkingConfig(thinking_budget=128) - elif m.startswith("gemini-2.5-flash"): - thinking = types.ThinkingConfig(thinking_budget=0) - elif "pro" in m: - thinking = types.ThinkingConfig(thinking_level="low") - else: - thinking = types.ThinkingConfig(thinking_level=_gemini_floor(model)) - else: - if m.startswith("gemini-2.5"): - thinking = types.ThinkingConfig(thinking_budget=-1) - else: - thinking = types.ThinkingConfig(thinking_level="high") - config = types.GenerateContentConfig( - temperature=DEFAULT_TEMPERATURE, - thinking_config=thinking, - ) - resp = client.models.generate_content( - model=model, - contents=[types.Part.from_text(text=prompt)], - config=config, - ) - return ( - (resp.text or "").strip(), - getattr(resp, "response_id", None), - _gemini_usage(resp), - ) - - -def build_client(provider: str): - if provider == "interfaze": - from openai import OpenAI - - api_key = os.getenv("INTERFAZE_API_KEY") - if not api_key: - raise RuntimeError("INTERFAZE_API_KEY missing from .env") - # Force the production endpoint — the project .env's OPENAI_BASE_URL - # may point at a dev/staging Cloudflare Worker, which we explicitly do - # not want for benchmark numbers. - return OpenAI(base_url="https://api.interfaze.ai/v1", api_key=api_key) - if provider == "openai": - from openai import OpenAI - - api_key = os.getenv("OPENAI_API_KEY") - if not api_key: - raise RuntimeError("OPENAI_API_KEY missing from .env") - # Force the real OpenAI endpoint — .env's OPENAI_BASE_URL points at interfaze. - return OpenAI(base_url="https://api.openai.com/v1", api_key=api_key) - if provider == "anthropic": - from anthropic import Anthropic - - api_key = os.getenv("ANTHROPIC_API_KEY") - if not api_key: - raise RuntimeError("ANTHROPIC_API_KEY missing from .env") - return Anthropic(api_key=api_key) - if provider == "gemini": - from google import genai - - api_key = ( - os.getenv("GEMINI_KEY") - or os.getenv("GEMINI_API_KEY") - or os.getenv("GOOGLE_API_KEY") - ) - if not api_key: - raise RuntimeError("GEMINI_KEY missing from .env") - return genai.Client(api_key=api_key) - if provider == "openrouter": - from openai import OpenAI - - api_key = ( - os.getenv("OPENROUTER_API_KEY") - or os.getenv("openrouter_api_key") - or os.getenv("OPENROUTER_KEY") - ) - if not api_key: - raise RuntimeError("OPENROUTER_API_KEY missing from .env") - return OpenAI(base_url="https://openrouter.ai/api/v1", api_key=api_key) - if provider == "fireworks": - from openai import OpenAI - - # .env in this repo may spell the key lowercase; accept either casing. - api_key = os.getenv("FIREWORKS_API_KEY") or os.getenv("fireworks_api_key") - if not api_key: - raise RuntimeError("FIREWORKS_API_KEY missing from .env") - return OpenAI(base_url="https://api.fireworks.ai/inference/v1", api_key=api_key) - raise ValueError(f"unknown provider: {provider}") - - -def get_call_fn(provider: str): - return { - "interfaze": call_interfaze, - "openai": call_openai, - "anthropic": call_anthropic, - "gemini": call_gemini, - "openrouter": call_openrouter, - "fireworks": call_fireworks, - }[provider] - - -def model_slug(model: str) -> str: - return re.sub(r"[^a-z0-9]+", "-", model.lower()).strip("-") - - -# --------------------------------------------------------------------------- -# Pipeline -# --------------------------------------------------------------------------- - - -async def process_sample( - sample: dict, - call_fn, - model: str, - rate_limiter: RateLimiter, - writer: JsonlWriter, - progress: dict, - provider: str, - client, -) -> dict | None: - prompt = PROMPT_TEMPLATE.format( - question=sample["question"], - a=sample["a"], - b=sample["b"], - c=sample["c"], - d=sample["d"], - ) - last_error: str | None = None - - for attempt in range(1, MAX_RETRIES + 1): - await rate_limiter.acquire() - start = time.perf_counter() - try: - content, request_id, usage = await asyncio.to_thread( - call_fn, prompt, model, client - ) - latency_ms = int((time.perf_counter() - start) * 1000) - if not content: - last_error = "empty response content" - raise RuntimeError(last_error) - - predicted = parse_answer(content) - correct = predicted == sample["answer"] - - record = { - "id": sample["id"], - "language": sample["language"], - "row_index": sample["row_index"], - "subject": sample["subject"], - "answer": sample["answer"], - "prediction": predicted, - "correct": correct, - "response": content, - "request_id": request_id, - "latency_ms": latency_ms, - "attempts": attempt, - "provider": provider, - "model": model, - "input_tokens": usage.get("input_tokens"), - "output_tokens": usage.get("output_tokens"), - "reasoning_tokens": usage.get("reasoning_tokens"), - } - await writer.append(record) - - progress["done"] += 1 - if correct: - progress["correct"] += 1 - if predicted is None: - progress["unparseable"] += 1 - rt = usage.get("reasoning_tokens") - ot = usage.get("output_tokens") - tqdm.write( - f"[{provider}/{model} {progress['done']}/{progress['total']}] " - f"{sample['language']} subj={sample['subject'][:18]:18} " - f"gold={sample['answer']} pred={predicted or '?'} " - f"{'OK' if correct else 'X '} latency={latency_ms}ms " - f"reasoning_tok={rt if rt is not None else '?'} out_tok={ot if ot is not None else '?'}" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - tqdm.write( - f"[{provider}/{model} error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(2 ** (attempt - 1)) - - progress["failed"] += 1 - tqdm.write( - f"[{provider}/{model} FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}" - ) - return None - - -def load_all_samples(languages: list[str], limit: int | None) -> list[dict]: - """Loads from openai/MMMLU (full) or opencompass/mmmlu_lite based on - DATASET_VARIANT global.""" - ds_id = DATASET_LITE_ID if DATASET_VARIANT == "lite" else DATASET_FULL_ID - all_samples: list[dict] = [] - for lang in languages: - print(f"Loading {ds_id}/{lang} (split=test)...") - ds = load_dataset_for_variant(DATASET_VARIANT, lang) - rows = [ - build_sample_for_variant(DATASET_VARIANT, dict(row), lang, i) - for i, row in enumerate(ds) - ] - if limit is not None: - rows = rows[:limit] - all_samples.extend(rows) - return all_samples - - -async def run( - provider: str, model: str, languages: list[str], pred_path: Path, limit: int | None -): - client = build_client(provider) - call_fn = get_call_fn(provider) - - samples = load_all_samples(languages, limit) - done_ids = load_completed_ids(pred_path) - pending = [s for s in samples if s["id"] not in done_ids] - print(f"[{provider}/{model}] Total samples: {len(samples)}") - print( - f"[{provider}/{model}] Resume: {len(done_ids)} done, {len(pending)} pending " - f"(checkpoint: {pred_path})" - ) - if not pending: - return - - writer = JsonlWriter(pred_path) - rate_limiter = RateLimiter(RATE_LIMIT) - progress = { - "total": len(pending), - "done": 0, - "correct": 0, - "unparseable": 0, - "failed": 0, - } - - sem = asyncio.Semaphore(MAX_IN_FLIGHT) - - async def bounded(sample): - async with sem: - return await process_sample( - sample, call_fn, model, rate_limiter, writer, progress, provider, client - ) - - tasks = [bounded(s) for s in pending] - try: - await tqdm_asyncio.gather(*tasks, desc=f"{provider}/{model}") - except Exception: - traceback.print_exc() - - acc = progress["correct"] / progress["done"] if progress["done"] else 0.0 - print( - f"\n[{provider}/{model}] Run finished: " - f"{progress['done']}/{progress['total']} answered " - f"({progress['failed']} failed, {progress['unparseable']} unparseable). " - f"Pooled accuracy on this run: {acc:.4f}" - ) - - -def run_evaluation(pred_path: Path, metrics_path: Path, provider: str, model: str): - if not pred_path.exists(): - print(f"No predictions found at {pred_path}") - sys.exit(1) - results = load_records(pred_path) - if not results: - print(f"No records in {pred_path}") - sys.exit(1) - for r in results: - if r.get("prediction") is None and r.get("response"): - r["prediction"] = parse_answer(r["response"]) - if r.get("correct") is None and r.get("prediction") is not None: - r["correct"] = r["prediction"] == r.get("answer") - - metrics = compute_metrics(results) - _base_print_summary(metrics) - output = { - **metrics, - "dataset": DATASET_ID, - "split": SPLIT, - "languages": sorted({r["language"] for r in results}), - "provider": provider, - "model": model, - "rate_limit": RATE_LIMIT, - "temperature": DEFAULT_TEMPERATURE, - } - metrics_path.parent.mkdir(parents=True, exist_ok=True) - with open(metrics_path, "w") as f: - json.dump(output, f, indent=2, ensure_ascii=False) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - global REASONING_MODE, DATASET_VARIANT - parser = argparse.ArgumentParser(description="Multi-provider MMMLU runner") - parser.add_argument( - "--provider", - required=True, - choices=[ - "interfaze", - "openai", - "anthropic", - "gemini", - "openrouter", - "fireworks", - ], - ) - parser.add_argument("--model", required=True, help="Provider-specific model id") - parser.add_argument( - "--reasoning", - default="off", - choices=["off", "high"], - help="off = each model at its floor; high = each at max thinking", - ) - parser.add_argument( - "--dataset-variant", - default="lite", - choices=["lite", "full"], - help="lite = opencompass/mmmlu_lite (~20k); full = openai/MMMLU (~196k)", - ) - parser.add_argument( - "--languages", - nargs="+", - default=LANGUAGES, - choices=LANGUAGES, - help="Subset of languages (default: all 14)", - ) - parser.add_argument( - "--limit", type=int, default=None, help="Cap samples per language (smoke test)" - ) - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - args = parser.parse_args() - - REASONING_MODE = args.reasoning - DATASET_VARIANT = args.dataset_variant - variant_slug = "lite" if DATASET_VARIANT == "lite" else "full" - tag = f"mmmlu{variant_slug}_{args.provider}_{model_slug(args.model)}_reasoning{REASONING_MODE}" - pred_path = RESULTS_DIR / f"{tag}_responses.jsonl" - metrics_path = RESULTS_DIR / f"{tag}_metrics.json" - - if args.evaluate_only: - run_evaluation(pred_path, metrics_path, args.provider, args.model) - elif args.predict_only: - asyncio.run( - run(args.provider, args.model, args.languages, pred_path, limit=args.limit) - ) - else: - asyncio.run( - run(args.provider, args.model, args.languages, pred_path, limit=args.limit) - ) - run_evaluation(pred_path, metrics_path, args.provider, args.model) - - -if __name__ == "__main__": - main() diff --git a/scripts/run_fireworks_suite.sh b/scripts/run_fireworks_suite.sh index 26c81fe..0cbbf81 100755 --- a/scripts/run_fireworks_suite.sh +++ b/scripts/run_fireworks_suite.sh @@ -87,8 +87,12 @@ run_step refcoco uv run -m benchmarks.obj_detection.refcoco_multi \ run_step olmocr uv run -m benchmarks.olmocr.olmocr_bench_fireworks --model "$MODEL" -run_step mmmlu uv run -m benchmarks.mmmlu.mmmlu_multi \ - --provider "$PROVIDER" --model "$MODEL" --reasoning "$REASONING" +if [ -n "${TARGET:-}" ]; then + run_step mmmlu uv run python -m bench_core run --benchmark mmmlu --target "$TARGET" \ + --reasoning "$([ "$REASONING" = "high" ] && echo high || echo off)" +else + echo " [skip] mmmlu: set TARGET= (python -m bench_core list-targets)" +fi run_step ocrbench uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks --model "$MODEL" \ --provider "$PROVIDER" --reasoning "$BENCH_OPENROUTER_EFFORT" diff --git a/tests/test_mmmlu_bench.py b/tests/test_mmmlu_bench.py new file mode 100644 index 0000000..9e0fa61 --- /dev/null +++ b/tests/test_mmmlu_bench.py @@ -0,0 +1,67 @@ +"""MMMLU: parse units + offline scoring-parity replay of the archived runs.""" + +import json +import pathlib + +import pytest + +from benchmarks.mmmlu import bench + +RESULTS = pathlib.Path(__file__).resolve().parent.parent / "results" +ARCHIVED_TAGS = [ + "mmmlulite_fireworks_accounts-fireworks-models-inkling_reasoningoff", + "mmmlulite_fireworks_accounts-fireworks-models-deepseek-v4-flash_reasoningoff", + "mmmlulite_gemini_gemini-3-7-flash_reasoninghigh", + "mmmlulite_openrouter_thinkingmachines-inkling-small_reasoninghigh", +] + + +def test_parse_answer(): + assert bench.parse_answer("B") == "B" + assert bench.parse_answer("The answer is (C).") == "C" + assert bench.parse_answer("") is None + + +def test_score_macro_averages_languages(): + samples = [ + {"id": "EN:0", "language": "EN", "subject": "math", "answer": "A"}, + {"id": "FR:0", "language": "FR", "subject": "math", "answer": "B"}, + ] + records = [{"id": "EN:0", "prediction": "A"}, {"id": "FR:0", "prediction": "C"}] + m = bench.score(records, samples) + assert m["per_language"]["EN"]["accuracy"] == 1.0 + assert m["per_language"]["FR"]["accuracy"] == 0.0 + assert m["macro_accuracy"] == 0.5 + + +@pytest.mark.parametrize("tag", ARCHIVED_TAGS) +def test_scoring_parity_with_archived_run(tag): + resp_path = RESULTS / f"{tag}_responses.jsonl" + metrics_path = RESULTS / f"{tag}_metrics.json" + if not (resp_path.exists() and metrics_path.exists()): + pytest.skip(f"archived run {tag} not present") + + archived = json.loads(metrics_path.read_text()) + samples, records = [], [] + for line in resp_path.read_text().splitlines(): + if not line.strip(): + continue + rec = json.loads(line) + if rec.get("response") is None: + continue + samples.append( + { + "id": rec["id"], + "language": rec["language"], + "subject": rec["subject"], + "answer": rec["answer"], + } + ) + records.append( + {"id": rec["id"], "prediction": bench.parse_answer(rec["response"])} + ) + + m = bench.score(records, samples) + assert m["num_samples"] == archived["num_samples"] + assert m["macro_accuracy"] == pytest.approx(archived["macro_accuracy"]) + assert m["micro_accuracy"] == pytest.approx(archived["micro_accuracy"]) From 747118243eb46ff6243d69a823697af6c8c2fc03 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 08:44:01 +0530 Subject: [PATCH 10/38] feat(bench_core): migrate MMMU-Pro onto the abstraction; retire fork + modal app MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit benchmarks/mmmu_pro/bench.py (standard + vision, A-J, per-subject/difficulty) replaces mmmu_pro_multi.py. CLI gains a generic --variant (standard|vision). Offline parity replays the 6 archived runs (standard inkling 0.7329, gemini 0.8486; vision inkling 0.6809, gemini 0.8324). Also deletes modal_app.py — it wrapped mmmu_pro_multi's internals for a Modal deployment and dies with the fork; Modal deployment, if wanted, should target the new CLI. 139 tests pass. --- README.md | 14 +- bench_core/cli.py | 28 +- benchmarks/mmmu_pro/bench.py | 155 +++++ benchmarks/mmmu_pro/mmmu_pro_multi.py | 913 -------------------------- benchmarks/mmmu_pro/modal_app.py | 259 -------- scripts/run_fireworks_suite.sh | 14 +- tests/test_mmmu_pro_bench.py | 62 ++ 7 files changed, 256 insertions(+), 1189 deletions(-) create mode 100644 benchmarks/mmmu_pro/bench.py delete mode 100644 benchmarks/mmmu_pro/mmmu_pro_multi.py delete mode 100644 benchmarks/mmmu_pro/modal_app.py create mode 100644 tests/test_mmmu_pro_bench.py diff --git a/README.md b/README.md index 899aa66..1a0f223 100644 --- a/README.md +++ b/README.md @@ -111,8 +111,8 @@ uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks uv run -m benchmarks.olmocr.olmocr_bench_fireworks uv run python -m bench_core run --target inkling --benchmark gpqa uv run python -m bench_core run --target inkling --benchmark mmmlu -uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider fireworks --model accounts/fireworks/models/inkling --setting standard -uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider fireworks --model accounts/fireworks/models/inkling --setting vision +uv run python -m bench_core run --target inkling --benchmark mmmu_pro --variant standard +uv run python -m bench_core run --target inkling --benchmark mmmu_pro --variant vision uv run -m benchmarks.obj_detection.refcoco_multi --provider fireworks --model accounts/fireworks/models/inkling uv run python -m bench_core run --target inkling --benchmark asr uv run -m benchmarks.spider2_lite.spider2_lite --provider fireworks @@ -246,12 +246,10 @@ Links: [paper](https://arxiv.org/abs/2409.02813) · [dataset](https://huggingfac Two settings: `standard` (text + inline images) and `vision` (rendered question image). ```bash -# Any provider, standard or vision -uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider gemini --model gemini-3.1-pro-preview --setting standard -uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider gemini --model gemini-3.1-pro-preview --setting vision -uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider openai --model gpt-5.5 --setting standard -uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider anthropic --model claude-sonnet-4-6 --setting vision -uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider interfaze --model interfaze-beta --setting standard +# Any target, standard or vision +uv run python -m bench_core run --target gemini-3.7-flash --benchmark mmmu_pro --variant standard +uv run python -m bench_core run --target gemini-3.7-flash --benchmark mmmu_pro --variant vision +uv run python -m bench_core run --target inkling --benchmark mmmu_pro --variant standard # Run on Modal instead of locally bash benchmarks/mmmu_pro/run_full.sh diff --git a/bench_core/cli.py b/bench_core/cli.py index 04ca252..0b3bfb2 100644 --- a/bench_core/cli.py +++ b/bench_core/cli.py @@ -20,6 +20,7 @@ "gpqa": "benchmarks.gpqa.bench", "asr": "benchmarks.asr.bench", "mmmlu": "benchmarks.mmmlu.bench", + "mmmu_pro": "benchmarks.mmmu_pro.bench", } @@ -41,9 +42,23 @@ def cmd_run(args) -> None: ) client = adapter.build_client() - print(f"Loading {args.benchmark} samples...") - samples = bench.load_samples(args.sample) - store = RunStore(bench.NAME, target.name) + variants = getattr(bench, "VARIANTS", None) + if variants: + variant = args.variant or variants[0] + if variant not in variants: + raise SystemExit( + f"--variant must be one of {variants} for {args.benchmark}" + ) + samples = bench.load_samples(args.sample, variant=variant) + result_name = f"{bench.NAME}_{variant}" + elif args.variant: + raise SystemExit(f"{args.benchmark} has no variants") + else: + samples = bench.load_samples(args.sample) + result_name = bench.NAME + + print(f"Loading {result_name} samples...") + store = RunStore(result_name, target.name) d = bench.DEFAULTS result = asyncio.run( @@ -65,7 +80,7 @@ def cmd_run(args) -> None: metrics = bench.score(store.load_responses(), samples) metrics.update( { - "benchmark": bench.NAME, + "benchmark": result_name, "target": target.name, "provider": target.provider, "model_id": target.model_id, @@ -119,6 +134,11 @@ def main(argv=None) -> None: r.add_argument( "--reasoning", default=None, help="off|low|medium|high (default: benchmark's)" ) + r.add_argument( + "--variant", + default=None, + help="benchmark variant, e.g. mmmu_pro: standard|vision", + ) r.add_argument( "--sample", type=int, default=None, help="run only the first N samples" ) diff --git a/benchmarks/mmmu_pro/bench.py b/benchmarks/mmmu_pro/bench.py new file mode 100644 index 0000000..e483e17 --- /dev/null +++ b/benchmarks/mmmu_pro/bench.py @@ -0,0 +1,155 @@ +"""MMMU-Pro: multimodal MCQ (A-J), standard (text+images) and vision settings.""" + +from __future__ import annotations + +import ast +import re +from collections import defaultdict + +from bench_core.media import encode_image +from bench_core.request import Message, ReasoningSpec, Request, TextPart + +NAME = "mmmu_pro" +ID_KEY = "id" +PRIMARY_METRIC = "accuracy" +VARIANTS = ["standard", "vision"] +DEFAULTS = {"reasoning": "off", "rate_limit": 25, "max_in_flight": 8} + +_DATASET_REPO = "MMMU/MMMU_Pro" +_SPLIT = "test" +_CONFIGS = {"standard": "standard (10 options)", "vision": "vision"} +_MAX_IMAGE_SIDE = 1536 +_LETTERS = list("ABCDEFGHIJ") + +_PROMPT_STANDARD = ( + "Answer the following multiple-choice question. The question may reference " + 'images via tags like "", "". The corresponding images ' + "are attached in order.\n\n" + "Respond with ONLY a single letter (A through J) corresponding to the correct " + "option. Do not explain.\n\n" + "Question: {question}\n\n" + "Options:\n{options}\n\n" + "Answer:" +) +_PROMPT_VISION = ( + "The attached image renders a multiple-choice question with its options. " + "Respond with ONLY a single letter (A through J) corresponding to the correct " + "option. Do not explain.\n\n" + "Answer:" +) + +_LETTER_RE = re.compile(r"\b([A-J])\b") +_FALLBACK_RE = re.compile(r"[A-J]") + + +def parse_answer(text: str) -> str | None: + if not text: + return None + s = text.strip() + if len(s) == 1 and s.upper() in _LETTERS: + return s.upper() + m = _LETTER_RE.search(s.upper()) + if m: + return m.group(1) + m = _FALLBACK_RE.search(s.upper()) + return m.group(0) if m else None + + +def _options(raw) -> list[str]: + return list(raw) if isinstance(raw, list) else list(ast.literal_eval(raw)) + + +def _options_block(options: list[str]) -> str: + return "\n".join(f"{ltr}. {opt}" for ltr, opt in zip(_LETTERS, options)) + + +def load_samples( + sample_size: int | None = None, variant: str = "standard" +) -> list[dict]: + from datasets import load_dataset + + ds = load_dataset(_DATASET_REPO, _CONFIGS[variant], split=_SPLIT) + rows = list(ds)[:sample_size] if sample_size else list(ds) + samples = [] + for row in rows: + row = dict(row) + if variant == "vision": + samples.append( + { + "id": row["id"], + "setting": "vision", + "image": row["image"], + "options": _options(row["options"]), + "answer": str(row["answer"]).strip().upper(), + "subject": row.get("subject"), + "topic_difficulty": None, + } + ) + else: + images = [row.get(f"image_{i}") for i in range(1, 8)] + samples.append( + { + "id": row["id"], + "setting": "standard", + "question": row["question"], + "options": _options(row["options"]), + "images": [im for im in images if im is not None], + "answer": str(row["answer"]).strip().upper(), + "subject": row.get("subject"), + "topic_difficulty": row.get("topic_difficulty"), + } + ) + return samples + + +def build_request(sample: dict, mode: str) -> Request: + if sample["setting"] == "vision": + prompt = _PROMPT_VISION + pil_images = [sample["image"]] + else: + prompt = _PROMPT_STANDARD.format( + question=sample["question"], options=_options_block(sample["options"]) + ) + pil_images = sample["images"] + parts = [TextPart(prompt)] + parts += [ + encode_image(im, "image/jpeg", max_side=_MAX_IMAGE_SIDE) for im in pil_images + ] + return Request([Message("user", parts)], ReasoningSpec(mode), temperature=0.0) + + +def parse(response, sample) -> str | None: + return parse_answer(response.text or "") + + +def score(records: list[dict], samples: list[dict]) -> dict: + by_id = {s["id"]: s for s in samples} + latest = {r["id"]: r for r in records if r["id"] in by_id} + + total = correct = unparse = 0 + by_subject: dict[str, list[bool]] = defaultdict(list) + by_difficulty: dict[str, list[bool]] = defaultdict(list) + for rid, r in latest.items(): + s = by_id[rid] + pred = r.get("prediction") + ok = pred == s["answer"] + total += 1 + correct += int(ok) + unparse += int(pred is None) + if s.get("subject") is not None: + by_subject[s["subject"]].append(ok) + if s.get("topic_difficulty") is not None: + by_difficulty[s["topic_difficulty"]].append(ok) + + return { + "accuracy": correct / total if total else 0.0, + "num_samples": total, + "unparseable": unparse, + "per_subject": { + k: {"n": len(v), "accuracy": sum(v) / len(v)} for k, v in by_subject.items() + }, + "per_difficulty": { + k: {"n": len(v), "accuracy": sum(v) / len(v)} + for k, v in by_difficulty.items() + }, + } diff --git a/benchmarks/mmmu_pro/mmmu_pro_multi.py b/benchmarks/mmmu_pro/mmmu_pro_multi.py deleted file mode 100644 index 293f718..0000000 --- a/benchmarks/mmmu_pro/mmmu_pro_multi.py +++ /dev/null @@ -1,913 +0,0 @@ -""" -MMMU Pro benchmark — multi-provider runner. - -Two settings, both 10-option MCQ (A-J): - - standard: text question + up to 7 inline images (image_1..image_7) - - vision: single rendered image of the entire question (image) - -Methodology mirrors the MMMU Pro paper: - - pass@1, single attempt, no majority voting - - reasoning OFF (provider floor) by default; --reasoning high also supported - - temperature=0 where the API allows it (some providers reject t=0 + thinking) - - headline numbers are per-setting accuracy; the published "MMMU Pro" score - is the average of (standard, vision) - -Output (per provider+model+setting+reasoning): - results/mmmupro____reasoning_responses.jsonl - results/mmmupro____reasoning_metrics.json - -Usage: - uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider gemini \\ - --model gemini-3.1-pro-preview --setting standard - uv run -m benchmarks.mmmu_pro.mmmu_pro_multi --provider openai \\ - --model gpt-5.5 --setting vision --limit 5 - -Env: OPENAI_API_KEY, GEMINI_KEY, ANTHROPIC_API_KEY, INTERFAZE_API_KEY (.env). -""" - -import os -import re -import io -import sys -import ast -import json -import time -import base64 -import asyncio -import argparse -import traceback -from pathlib import Path -from collections import defaultdict - -from datasets import load_dataset -from dotenv import load_dotenv -from PIL import Image -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -load_dotenv() - - -def _load_interfaze_env_fallback() -> None: - path = Path.home() / "interfaze" / ".env.local" - if not path.exists(): - return - for line in path.read_text().splitlines(): - line = line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - k, v = line.split("=", 1) - os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) - - -_load_interfaze_env_fallback() - - -RESULTS_DIR = PROJECT_ROOT / "results" -DATASET_REPO = "MMMU/MMMU_Pro" -SETTINGS = { - "standard": "standard (10 options)", - "vision": "vision", -} -SPLIT = "test" - -REASONING_MODE = "off" -DEFAULT_TEMPERATURE = 0.0 -RATE_LIMIT = 25 -MAX_RETRIES = 3 - -# Cap on requests in flight. RATE_LIMIT bounds how many requests *start* per -# second but not how many are outstanding: gather() launches every pending -# sample at once, so hundreds pile up, queue server-side, and eventually 429 -# after tens of minutes of latency. Override with BENCH_MAX_IN_FLIGHT. -MAX_IN_FLIGHT = int(os.getenv("BENCH_MAX_IN_FLIGHT", "8")) - -def _gemini_floor(model: str) -> str: - """Lowest thinking_level the given Gemini model actually accepts. - - gemini-3.7+ flash rejects MINIMAL server-side ("Thinking level MINIMAL is - not supported for this model"), and the SDK enum has no "off"/"none", so - "low" is its true floor. Older 3.x flash models still take "minimal". - BENCH_GEMINI_FLOOR overrides for anything not covered here. - """ - override = os.getenv("BENCH_GEMINI_FLOOR") - if override: - return override - m = model.lower() - if any(v in m for v in ("3.7", "3.8", "3.9", "4.")): - return "low" - return "minimal" - -ANTHROPIC_OFF_MAX_TOKENS = 1024 -ANTHROPIC_HIGH_BUDGET_TOKENS = 10_000 -ANTHROPIC_HIGH_MAX_TOKENS = 16_000 - -# Cap any individual image side to keep token cost in check; questions occasionally -# include large diagrams. 1536 keeps detail without paying for >2k-side renders. -MAX_IMAGE_SIDE = 1536 - -LETTERS_10 = list("ABCDEFGHIJ") - - -# --------------------------------------------------------------------------- -# IO helpers -# --------------------------------------------------------------------------- - - -class JsonlWriter: - def __init__(self, path: Path): - self.path = path - self.path.parent.mkdir(parents=True, exist_ok=True) - self._lock = asyncio.Lock() - - async def append(self, record: dict): - line = json.dumps(record, ensure_ascii=False) - async with self._lock: - with open(self.path, "a", encoding="utf-8") as f: - f.write(line + "\n") - f.flush() - os.fsync(f.fileno()) - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def load_completed_ids(path: Path) -> set[str]: - if not path.exists(): - return set() - done: set[str] = set() - with open(path, encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - rec = json.loads(line) - if rec.get("response") is not None: - done.add(str(rec["id"])) - except json.JSONDecodeError: - continue - return done - - -def load_records(path: Path) -> list[dict]: - if not path.exists(): - return [] - by_id: dict[str, dict] = {} - with open(path, encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - rec = json.loads(line) - by_id[str(rec["id"])] = rec - except json.JSONDecodeError: - continue - return list(by_id.values()) - - -# --------------------------------------------------------------------------- -# Dataset / sample building -# --------------------------------------------------------------------------- - - -def parse_options(raw: str) -> list[str]: - """`options` is a stringified Python list. Use literal_eval (safe).""" - if isinstance(raw, list): - return list(raw) - return list(ast.literal_eval(raw)) - - -def build_standard_sample(row: dict) -> dict: - images: list[Image.Image] = [] - for i in range(1, 8): - img = row.get(f"image_{i}") - if img is not None: - images.append(img) - return { - "id": row["id"], - "setting": "standard", - "question": row["question"], - "options": parse_options(row["options"]), - "images": images, - "answer": str(row["answer"]).strip().upper(), - "subject": row.get("subject"), - "topic_difficulty": row.get("topic_difficulty"), - } - - -def build_vision_sample(row: dict) -> dict: - return { - "id": row["id"], - "setting": "vision", - "image": row["image"], - "options": parse_options(row["options"]), - "answer": str(row["answer"]).strip().upper(), - "subject": row.get("subject"), - } - - -def load_samples(setting: str, limit: int | None) -> list[dict]: - cfg = SETTINGS[setting] - print(f"Loading {DATASET_REPO} config={cfg!r} split={SPLIT}...") - ds = load_dataset(DATASET_REPO, cfg, split=SPLIT) - rows = list(ds) - if limit is not None: - rows = rows[:limit] - if setting == "standard": - return [build_standard_sample(dict(r)) for r in rows] - return [build_vision_sample(dict(r)) for r in rows] - - -# --------------------------------------------------------------------------- -# Image preprocessing -# --------------------------------------------------------------------------- - - -def image_to_jpeg_bytes(image: Image.Image) -> bytes: - if image.mode != "RGB": - image = image.convert("RGB") - w, h = image.size - scale = min(1.0, MAX_IMAGE_SIDE / max(w, h)) - if scale < 1.0: - image = image.resize((int(round(w * scale)), int(round(h * scale)))) - buf = io.BytesIO() - image.save(buf, format="JPEG", quality=92) - return buf.getvalue() - - -# --------------------------------------------------------------------------- -# Prompt construction -# --------------------------------------------------------------------------- - - -def build_options_block(options: list[str]) -> str: - lines = [] - for letter, opt in zip(LETTERS_10, options): - lines.append(f"{letter}. {opt}") - return "\n".join(lines) - - -PROMPT_STANDARD = ( - "Answer the following multiple-choice question. The question may reference " - 'images via tags like "", "". The corresponding images ' - "are attached in order.\n\n" - "Respond with ONLY a single letter (A through J) corresponding to the correct " - "option. Do not explain.\n\n" - "Question: {question}\n\n" - "Options:\n{options}\n\n" - "Answer:" -) - -PROMPT_VISION = ( - "The attached image renders a multiple-choice question with its options. " - "Respond with ONLY a single letter (A through J) corresponding to the correct " - "option. Do not explain.\n\n" - "Answer:" -) - - -# --------------------------------------------------------------------------- -# Output parsing -# --------------------------------------------------------------------------- - -_LETTER_RE = re.compile(r"\b([A-J])\b") -_FALLBACK_RE = re.compile(r"[A-J]") - - -def parse_answer(text: str) -> str | None: - if not text: - return None - s = text.strip() - if len(s) == 1 and s.upper() in LETTERS_10: - return s.upper() - m = _LETTER_RE.search(s.upper()) - if m: - return m.group(1) - m = _FALLBACK_RE.search(s.upper()) - if m: - return m.group(0) - return None - - -# --------------------------------------------------------------------------- -# Provider adapters -# --------------------------------------------------------------------------- - - -def _safe_int(x): - try: - return int(x) if x is not None else None - except (TypeError, ValueError): - return None - - -def _openai_usage(resp) -> dict: - u = getattr(resp, "usage", None) - if u is None: - return {"input_tokens": None, "output_tokens": None, "reasoning_tokens": None} - details = getattr(u, "completion_tokens_details", None) - rt = getattr(details, "reasoning_tokens", None) if details else None - if rt is None: - # Fireworks reports it at the top level of `usage` for Inkling. - rt = getattr(u, "reasoning_tokens", None) - return { - "input_tokens": _safe_int(getattr(u, "prompt_tokens", None)), - "output_tokens": _safe_int(getattr(u, "completion_tokens", None)), - "reasoning_tokens": _safe_int(rt), - } - - -def _gemini_usage(resp) -> dict: - u = getattr(resp, "usage_metadata", None) - if u is None: - return {"input_tokens": None, "output_tokens": None, "reasoning_tokens": None} - return { - "input_tokens": _safe_int(getattr(u, "prompt_token_count", None)), - "output_tokens": _safe_int(getattr(u, "candidates_token_count", None)), - "reasoning_tokens": _safe_int(getattr(u, "thoughts_token_count", None)), - } - - -def _anthropic_usage(resp) -> dict: - u = getattr(resp, "usage", None) - if u is None: - return {"input_tokens": None, "output_tokens": None, "reasoning_tokens": None} - return { - "input_tokens": _safe_int(getattr(u, "input_tokens", None)), - "output_tokens": _safe_int(getattr(u, "output_tokens", None)), - "reasoning_tokens": None, - } - - -def _images_for_sample(sample: dict) -> list[Image.Image]: - if sample["setting"] == "vision": - return [sample["image"]] - return sample["images"] - - -def _build_prompt(sample: dict) -> str: - if sample["setting"] == "vision": - return PROMPT_VISION - return PROMPT_STANDARD.format( - question=sample["question"], - options=build_options_block(sample["options"]), - ) - - -def _openai_style_content(prompt: str, images: list[Image.Image]) -> list[dict]: - """OpenAI/Interfaze chat content blocks: text + image_url(data URL).""" - content: list[dict] = [{"type": "text", "text": prompt}] - for img in images: - b64 = base64.b64encode(image_to_jpeg_bytes(img)).decode("utf-8") - content.append( - { - "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64}"}, - } - ) - return content - - -def call_interfaze(sample: dict, model: str, client) -> tuple[str, str | None, dict]: - content = _openai_style_content(_build_prompt(sample), _images_for_sample(sample)) - kwargs = { - "model": model, - "messages": [{"role": "user", "content": content}], - "temperature": DEFAULT_TEMPERATURE, - "reasoning_effort": "off" if REASONING_MODE == "off" else "high", - } - resp = client.chat.completions.create(**kwargs) - return ( - (resp.choices[0].message.content or "").strip(), - getattr(resp, "id", None), - _openai_usage(resp), - ) - - -def call_openai(sample: dict, model: str, client) -> tuple[str, str | None, dict]: - content = _openai_style_content(_build_prompt(sample), _images_for_sample(sample)) - kwargs = { - "model": model, - "messages": [{"role": "user", "content": content}], - } - if REASONING_MODE == "off": - kwargs["reasoning_effort"] = "none" - kwargs["temperature"] = DEFAULT_TEMPERATURE - else: - kwargs["reasoning_effort"] = "high" - resp = client.chat.completions.create(**kwargs) - return ( - (resp.choices[0].message.content or "").strip(), - getattr(resp, "id", None), - _openai_usage(resp), - ) - - -def call_anthropic(sample: dict, model: str, client) -> tuple[str, str | None, dict]: - prompt = _build_prompt(sample) - parts: list[dict] = [] - for img in _images_for_sample(sample): - b64 = base64.b64encode(image_to_jpeg_bytes(img)).decode("utf-8") - parts.append( - { - "type": "image", - "source": {"type": "base64", "media_type": "image/jpeg", "data": b64}, - } - ) - parts.append({"type": "text", "text": prompt}) - kwargs = { - "model": model, - "messages": [{"role": "user", "content": parts}], - } - if REASONING_MODE == "off": - kwargs["thinking"] = {"type": "disabled"} - kwargs["temperature"] = DEFAULT_TEMPERATURE - kwargs["max_tokens"] = ANTHROPIC_OFF_MAX_TOKENS - else: - kwargs["thinking"] = { - "type": "enabled", - "budget_tokens": ANTHROPIC_HIGH_BUDGET_TOKENS, - } - kwargs["max_tokens"] = ANTHROPIC_HIGH_MAX_TOKENS - resp = client.messages.create(**kwargs) - text_parts = [b.text for b in resp.content if getattr(b, "type", None) == "text"] - return "\n".join(text_parts).strip(), resp.id, _anthropic_usage(resp) - - -def _openrouter_extra_body(model: str) -> dict: - """Per-model OpenRouter extras. See benchmarks.mmmlu.mmmlu_multi for details. - Vision-capable models we currently route: x-ai/grok-4.3, moonshotai/kimi-k2.6.""" - m = model.lower() - if m.startswith("x-ai/grok-4.3"): - if REASONING_MODE == "off": - return {"reasoning": {"effort": "minimal"}} - return {"reasoning": {"enabled": True}} - if m.startswith("google/"): - # Gemini rejects reasoning.enabled=false outright ("Reasoning is - # mandatory for this endpoint and cannot be disabled"), so "off" maps to - # its real floor: effort=minimal, measured at 0 reasoning tokens. This - # matches what call_gemini does natively (thinking_level="minimal"). - return {"reasoning": {"effort": "minimal" if REASONING_MODE == "off" else "high"}} - if m.startswith("moonshotai/"): - body = {"reasoning": {"enabled": REASONING_MODE != "off"}} - body["provider"] = {"only": ["moonshotai"]} - return body - return {"reasoning": {"enabled": REASONING_MODE != "off"}} - - -def call_openrouter(sample: dict, model: str, client) -> tuple[str, str | None, dict]: - """OpenRouter dispatch (vision-aware). Vision: pass images as data: URLs - (OpenAI-shaped). See _openrouter_extra_body for per-model knobs.""" - content = _openai_style_content(_build_prompt(sample), _images_for_sample(sample)) - resp = client.chat.completions.create( - model=model, - messages=[{"role": "user", "content": content}], - temperature=DEFAULT_TEMPERATURE, - extra_body=_openrouter_extra_body(model), - ) - return ( - (resp.choices[0].message.content or "").strip(), - getattr(resp, "id", None), - _openai_usage(resp), - ) - - -def call_fireworks(sample: dict, model: str, client) -> tuple[str, str | None, dict]: - """Fireworks-hosted models (Inkling). Images go in as OpenAI-shaped - image_url data URLs. Inkling always thinks — "none" is the reasoning floor, - not a true off; per-sample reasoning_tokens are recorded so the actual - thinking spend is auditable.""" - content = _openai_style_content(_build_prompt(sample), _images_for_sample(sample)) - resp = client.chat.completions.create( - model=model, - messages=[{"role": "user", "content": content}], - temperature=DEFAULT_TEMPERATURE, - reasoning_effort="none" if REASONING_MODE == "off" else "high", - ) - return ( - (resp.choices[0].message.content or "").strip(), - getattr(resp, "id", None), - _openai_usage(resp), - ) - - -def call_gemini(sample: dict, model: str, client) -> tuple[str, str | None, dict]: - from google.genai import types - - prompt = _build_prompt(sample) - m = model.lower() - if REASONING_MODE == "off": - if m.startswith("gemini-2.5-pro"): - thinking = types.ThinkingConfig(thinking_budget=128) - elif m.startswith("gemini-2.5-flash"): - thinking = types.ThinkingConfig(thinking_budget=0) - elif "pro" in m: - thinking = types.ThinkingConfig(thinking_level="low") - else: - thinking = types.ThinkingConfig(thinking_level=_gemini_floor(model)) - else: - if m.startswith("gemini-2.5"): - thinking = types.ThinkingConfig(thinking_budget=-1) - else: - thinking = types.ThinkingConfig(thinking_level="high") - config = types.GenerateContentConfig( - temperature=DEFAULT_TEMPERATURE, - thinking_config=thinking, - ) - contents: list = [] - for img in _images_for_sample(sample): - contents.append( - types.Part.from_bytes(data=image_to_jpeg_bytes(img), mime_type="image/jpeg") - ) - contents.append(prompt) - resp = client.models.generate_content(model=model, contents=contents, config=config) - return ( - (resp.text or "").strip(), - getattr(resp, "response_id", None), - _gemini_usage(resp), - ) - - -def build_client(provider: str): - if provider == "interfaze": - from openai import OpenAI - - api_key = os.getenv("INTERFAZE_API_KEY") - if not api_key: - raise RuntimeError("INTERFAZE_API_KEY missing") - return OpenAI(base_url="https://api.interfaze.ai/v1", api_key=api_key) - if provider == "openai": - from openai import OpenAI - - api_key = os.getenv("OPENAI_API_KEY") - if not api_key: - raise RuntimeError("OPENAI_API_KEY missing") - return OpenAI(base_url="https://api.openai.com/v1", api_key=api_key) - if provider == "anthropic": - from anthropic import Anthropic - - api_key = os.getenv("ANTHROPIC_API_KEY") - if not api_key: - raise RuntimeError("ANTHROPIC_API_KEY missing") - return Anthropic(api_key=api_key) - if provider == "gemini": - from google import genai - - api_key = ( - os.getenv("GEMINI_KEY") - or os.getenv("GEMINI_API_KEY") - or os.getenv("GOOGLE_API_KEY") - ) - if not api_key: - raise RuntimeError("GEMINI_KEY missing") - return genai.Client(api_key=api_key) - if provider == "openrouter": - from openai import OpenAI - - api_key = ( - os.getenv("OPENROUTER_API_KEY") - or os.getenv("openrouter_api_key") - or os.getenv("OPENROUTER_KEY") - ) - if not api_key: - raise RuntimeError("OPENROUTER_API_KEY missing") - return OpenAI(base_url="https://openrouter.ai/api/v1", api_key=api_key) - if provider == "fireworks": - from openai import OpenAI - - # .env in this repo may spell the key lowercase; accept either casing. - api_key = os.getenv("FIREWORKS_API_KEY") or os.getenv("fireworks_api_key") - if not api_key: - raise RuntimeError("FIREWORKS_API_KEY missing") - return OpenAI(base_url="https://api.fireworks.ai/inference/v1", api_key=api_key) - raise ValueError(f"unknown provider: {provider}") - - -def get_call_fn(provider: str): - return { - "interfaze": call_interfaze, - "openai": call_openai, - "anthropic": call_anthropic, - "gemini": call_gemini, - "openrouter": call_openrouter, - "fireworks": call_fireworks, - }[provider] - - -def model_slug(model: str) -> str: - return re.sub(r"[^a-z0-9]+", "-", model.lower()).strip("-") - - -# --------------------------------------------------------------------------- -# Pipeline -# --------------------------------------------------------------------------- - - -async def process_sample( - sample: dict, - call_fn, - model: str, - rate_limiter: RateLimiter, - writer: JsonlWriter, - progress: dict, - provider: str, - client, -) -> dict | None: - last_error: str | None = None - - for attempt in range(1, MAX_RETRIES + 1): - await rate_limiter.acquire() - start = time.perf_counter() - try: - content, request_id, usage = await asyncio.to_thread( - call_fn, sample, model, client - ) - latency_ms = int((time.perf_counter() - start) * 1000) - if not content: - last_error = "empty response content" - raise RuntimeError(last_error) - - predicted = parse_answer(content) - correct = predicted == sample["answer"] - - record = { - "id": sample["id"], - "setting": sample["setting"], - "subject": sample.get("subject"), - "topic_difficulty": sample.get("topic_difficulty"), - "answer": sample["answer"], - "prediction": predicted, - "correct": correct, - "response": content, - "request_id": request_id, - "latency_ms": latency_ms, - "attempts": attempt, - "provider": provider, - "model": model, - "input_tokens": usage.get("input_tokens"), - "output_tokens": usage.get("output_tokens"), - "reasoning_tokens": usage.get("reasoning_tokens"), - } - await writer.append(record) - - progress["done"] += 1 - if correct: - progress["correct"] += 1 - if predicted is None: - progress["unparseable"] += 1 - rt = usage.get("reasoning_tokens") - ot = usage.get("output_tokens") - tqdm.write( - f"[{provider}/{model} {sample['setting']} {progress['done']}/{progress['total']}] " - f"id={sample['id']} subj={(sample.get('subject') or '')[:18]:18} " - f"gold={sample['answer']} pred={predicted or '?'} " - f"{'OK' if correct else 'X '} latency={latency_ms}ms " - f"reasoning_tok={rt if rt is not None else '?'} out_tok={ot if ot is not None else '?'}" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - tqdm.write( - f"[{provider}/{model} error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(2 ** (attempt - 1)) - - progress["failed"] += 1 - tqdm.write( - f"[{provider}/{model} FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}" - ) - return None - - -async def run( - provider: str, model: str, setting: str, pred_path: Path, limit: int | None -): - client = build_client(provider) - call_fn = get_call_fn(provider) - - samples = load_samples(setting, limit) - done_ids = load_completed_ids(pred_path) - pending = [s for s in samples if s["id"] not in done_ids] - print( - f"[{provider}/{model} {setting}] Total: {len(samples)}, " - f"resume: {len(done_ids)} done / {len(pending)} pending " - f"(checkpoint: {pred_path})" - ) - if not pending: - return - - writer = JsonlWriter(pred_path) - rate_limiter = RateLimiter(RATE_LIMIT) - progress = { - "total": len(pending), - "done": 0, - "correct": 0, - "unparseable": 0, - "failed": 0, - } - - sem = asyncio.Semaphore(MAX_IN_FLIGHT) - - async def bounded(sample): - async with sem: - return await process_sample( - sample, call_fn, model, rate_limiter, writer, progress, provider, client - ) - - tasks = [bounded(s) for s in pending] - try: - await tqdm_asyncio.gather(*tasks, desc=f"{provider}/{model}/{setting}") - except Exception: - traceback.print_exc() - - acc = progress["correct"] / progress["done"] if progress["done"] else 0.0 - print( - f"\n[{provider}/{model} {setting}] Run finished: " - f"{progress['done']}/{progress['total']} answered " - f"({progress['failed']} failed, {progress['unparseable']} unparseable). " - f"Pooled accuracy on this run: {acc:.4f}" - ) - - -# --------------------------------------------------------------------------- -# Metrics -# --------------------------------------------------------------------------- - - -def compute_metrics(results: list[dict]) -> dict: - if not results: - return {} - by_subject: dict[str, list[dict]] = defaultdict(list) - by_difficulty: dict[str, list[dict]] = defaultdict(list) - for r in results: - if r.get("subject") is not None: - by_subject[r["subject"]].append(r) - if r.get("topic_difficulty") is not None: - by_difficulty[r["topic_difficulty"]].append(r) - - n_total = len(results) - n_correct = sum(1 for r in results if r.get("correct")) - n_unparseable = sum(1 for r in results if r.get("prediction") is None) - - per_subject = {} - for subj, rows in by_subject.items(): - n = len(rows) - c = sum(1 for r in rows if r.get("correct")) - per_subject[subj] = {"n": n, "accuracy": c / n if n else 0.0} - - per_difficulty = {} - for diff, rows in by_difficulty.items(): - n = len(rows) - c = sum(1 for r in rows if r.get("correct")) - per_difficulty[diff] = {"n": n, "accuracy": c / n if n else 0.0} - - latencies = [ - r["latency_ms"] for r in results if isinstance(r.get("latency_ms"), int) - ] - latency_stats = {} - if latencies: - lats = sorted(latencies) - n = len(lats) - latency_stats = { - "count": n, - "mean_ms": sum(lats) / n, - "p50_ms": lats[n // 2], - "p90_ms": lats[min(n - 1, int(n * 0.9))], - "p99_ms": lats[min(n - 1, int(n * 0.99))], - "max_ms": lats[-1], - } - - return { - "accuracy": n_correct / n_total if n_total else 0.0, - "num_samples": n_total, - "unparseable": n_unparseable, - "per_subject": per_subject, - "per_difficulty": per_difficulty, - "latency": latency_stats, - } - - -def print_summary(metrics: dict, setting: str, provider: str, model: str): - print(f"\n{'=' * 68}") - print( - f"MMMU Pro [{setting}] — {provider}/{model} (reasoning={REASONING_MODE}, temp={DEFAULT_TEMPERATURE})" - ) - print(f"{'=' * 68}") - print(f"Samples : {metrics['num_samples']}") - print(f"Accuracy : {metrics['accuracy']:.4f}") - print(f"Unparseable : {metrics['unparseable']}") - if metrics.get("latency"): - lat = metrics["latency"] - print( - f"Latency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " - f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms" - ) - - -def run_evaluation( - pred_path: Path, metrics_path: Path, provider: str, model: str, setting: str -): - if not pred_path.exists(): - print(f"No predictions found at {pred_path}") - return - results = load_records(pred_path) - if not results: - print(f"No records in {pred_path}") - return - for r in results: - if r.get("prediction") is None and r.get("response"): - r["prediction"] = parse_answer(r["response"]) - if r.get("correct") is None and r.get("prediction") is not None: - r["correct"] = r["prediction"] == r.get("answer") - - metrics = compute_metrics(results) - print_summary(metrics, setting, provider, model) - output = { - **metrics, - "dataset": DATASET_REPO, - "config": SETTINGS[setting], - "setting": setting, - "split": SPLIT, - "provider": provider, - "model": model, - "reasoning_mode": REASONING_MODE, - "rate_limit": RATE_LIMIT, - "temperature": DEFAULT_TEMPERATURE, - } - metrics_path.parent.mkdir(parents=True, exist_ok=True) - with open(metrics_path, "w") as f: - json.dump(output, f, indent=2, ensure_ascii=False) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - global REASONING_MODE - parser = argparse.ArgumentParser(description="MMMU Pro multi-provider runner") - parser.add_argument( - "--provider", - required=True, - choices=[ - "interfaze", - "openai", - "anthropic", - "gemini", - "openrouter", - "fireworks", - ], - ) - parser.add_argument("--model", required=True) - parser.add_argument("--setting", required=True, choices=list(SETTINGS.keys())) - parser.add_argument("--reasoning", default="off", choices=["off", "high"]) - parser.add_argument( - "--limit", type=int, default=None, help="Cap samples (smoke test)" - ) - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - args = parser.parse_args() - - REASONING_MODE = args.reasoning - tag = f"mmmupro_{args.setting}_{args.provider}_{model_slug(args.model)}_reasoning{REASONING_MODE}" - pred_path = RESULTS_DIR / f"{tag}_responses.jsonl" - metrics_path = RESULTS_DIR / f"{tag}_metrics.json" - - if args.evaluate_only: - run_evaluation(pred_path, metrics_path, args.provider, args.model, args.setting) - elif args.predict_only: - asyncio.run( - run(args.provider, args.model, args.setting, pred_path, limit=args.limit) - ) - else: - asyncio.run( - run(args.provider, args.model, args.setting, pred_path, limit=args.limit) - ) - run_evaluation(pred_path, metrics_path, args.provider, args.model, args.setting) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/mmmu_pro/modal_app.py b/benchmarks/mmmu_pro/modal_app.py deleted file mode 100644 index 2721305..0000000 --- a/benchmarks/mmmu_pro/modal_app.py +++ /dev/null @@ -1,259 +0,0 @@ -""" -Modal runner for the MMMU Pro benchmark. - -Persists JSONL results + HF dataset cache to Modal Volumes so runs survive -local-machine death and can be inspected mid-flight without disturbing the -running container. - -Two settings, both 1730 samples each: - - standard: text question + up to 7 inline images - - vision: single rendered image of the entire question - -Quick reference (run from repo root): - - # smoke test (5 samples, attached so you see logs): - uv run modal run benchmarks/mmmu_pro/modal_app.py::run \\ - --provider gemini --model gemini-2.5-flash --setting standard --limit 5 - - # full detached run (survives Ctrl-C / laptop closing): - uv run modal run --detach benchmarks/mmmu_pro/modal_app.py::run \\ - --provider gemini --model gemini-3.1-pro-preview --setting standard - - # check progress (reads volume; does NOT touch the running container): - uv run modal run benchmarks/mmmu_pro/modal_app.py::check \\ - --provider gemini --model gemini-3.1-pro-preview --setting standard - - # pull results back to ./results/ when done: - uv run modal run benchmarks/mmmu_pro/modal_app.py::download \\ - --provider gemini --model gemini-3.1-pro-preview --setting standard - - # one-off prefetch of both dataset configs into the HF cache volume - # (recommended before launching parallel runs to avoid HF 429s): - uv run modal run benchmarks/mmmu_pro/modal_app.py::prefetch -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import modal - -APP_NAME = "interfaze-mmmu-pro" -SECRET_NAME = "screenspot-bench" # same provider keys; reused -RESULTS_VOLUME = "mmmu-pro-results" -HF_CACHE_VOLUME = "mmmu-pro-hf-cache" - -REPO_ROOT = Path(__file__).resolve().parent.parent.parent - -image = ( - modal.Image.debian_slim(python_version="3.12") - .pip_install( - "openai>=2.26.0", - "anthropic>=0.96.0", - "google-genai>=1.73.1", - "huggingface_hub>=0.30.0", - "datasets>=4.7.0", - "pillow>=10.0.0", - "python-dotenv>=1.2.2", - "tqdm>=4.66.0", - ) - .env({"PYTHONPATH": "/app", "HF_HOME": "/hf_cache"}) - # add_local_* must come last (Modal injects these at container startup, - # so they can't be followed by build steps). - .add_local_dir( - str(REPO_ROOT / "benchmarks" / "mmmu_pro"), - remote_path="/app/benchmarks/mmmu_pro", - ) - .add_local_file( - str(REPO_ROOT / "benchmarks" / "__init__.py"), - remote_path="/app/benchmarks/__init__.py", - ) -) - -results_vol = modal.Volume.from_name(RESULTS_VOLUME, create_if_missing=True) -hf_cache_vol = modal.Volume.from_name(HF_CACHE_VOLUME, create_if_missing=True) -secret = modal.Secret.from_name(SECRET_NAME) - -app = modal.App(APP_NAME) - - -def _slug(model: str) -> str: - import re - return re.sub(r"[^a-z0-9]+", "-", model.lower()).strip("-") - - -def _tag(provider: str, model: str, setting: str, reasoning: str) -> str: - return f"mmmupro_{setting}_{provider}_{_slug(model)}_reasoning{reasoning}" - - -# --------------------------------------------------------------------------- -# Prefetch: cache both dataset configs into the volume sequentially, so the -# parallel benchmark runs hit the local cache rather than HF directly. -# --------------------------------------------------------------------------- - -@app.function( - image=image, - volumes={"/hf_cache": hf_cache_vol}, - timeout=60 * 60 * 2, -) -def prefetch() -> None: - from datasets import load_dataset - for cfg in ["standard (10 options)", "vision"]: - print(f"Loading MMMU/MMMU_Pro config={cfg!r} split=test...") - ds = load_dataset("MMMU/MMMU_Pro", cfg, split="test") - print(f" cached {len(ds)} rows") - hf_cache_vol.commit() - - -# --------------------------------------------------------------------------- -# Run: execute the benchmark. Streams JSONL into the results volume, with -# fsync after every record (already done in mmmu_pro_multi.JsonlWriter), -# so a separate reader function sees up-to-date progress on volume.reload(). -# --------------------------------------------------------------------------- - -@app.function( - image=image, - volumes={"/results": results_vol, "/hf_cache": hf_cache_vol}, - secrets=[secret], - timeout=60 * 60 * 24, # 24h ceiling - cpu=4.0, - memory=8192, -) -def _run_remote(provider: str, model: str, setting: str, reasoning: str, - limit: int | None) -> dict: - import asyncio - import sys - sys.path.insert(0, "/app") - - # Override RESULTS_DIR to point at the mounted volume before importing. - import benchmarks.mmmu_pro.mmmu_pro_multi as bench - bench.RESULTS_DIR = Path("/results") - bench.REASONING_MODE = reasoning - - tag = _tag(provider, model, setting, reasoning) - pred_path = Path("/results") / f"{tag}_responses.jsonl" - metrics_path = Path("/results") / f"{tag}_metrics.json" - - # Periodic volume commits so the reader function sees progress. - async def _committer(): - while True: - await asyncio.sleep(20) - try: - results_vol.commit() - except Exception: - pass - - async def _main(): - committer = asyncio.create_task(_committer()) - try: - await bench.run(provider, model, setting, pred_path, limit=limit) - finally: - committer.cancel() - results_vol.commit() - - asyncio.run(_main()) - bench.run_evaluation(pred_path, metrics_path, provider, model, setting) - results_vol.commit() - - metrics = json.loads(metrics_path.read_text()) if metrics_path.exists() else {} - return {"tag": tag, "metrics_summary": { - k: metrics.get(k) for k in ("accuracy", "num_samples", "unparseable") - }} - - -@app.local_entrypoint() -def run(provider: str, model: str, setting: str = "standard", - reasoning: str = "off", limit: int | None = None): - """Run the benchmark. Use `modal run --detach ...` for unattended runs.""" - if setting not in ("standard", "vision"): - raise SystemExit(f"--setting must be 'standard' or 'vision', got {setting!r}") - result = _run_remote.remote(provider, model, setting, reasoning, limit) - print(json.dumps(result, indent=2)) - - -# --------------------------------------------------------------------------- -# Check: ephemeral, read-only progress probe. Does NOT touch the run container. -# --------------------------------------------------------------------------- - -@app.function( - image=image, - volumes={"/results": results_vol}, - timeout=120, -) -def _check_remote(provider: str, model: str, setting: str, reasoning: str) -> dict: - import sys - sys.path.insert(0, "/app") - from benchmarks.mmmu_pro.mmmu_pro_multi import ( - load_records, compute_metrics, - ) - - # Force-refresh the local view of the volume so we see whatever the - # running writer has flushed so far. - results_vol.reload() - - tag = _tag(provider, model, setting, reasoning) - pred_path = Path("/results") / f"{tag}_responses.jsonl" - if not pred_path.exists(): - return {"tag": tag, "status": "no-file-yet", "path": str(pred_path)} - - records = load_records(pred_path) - metrics = compute_metrics(records) if records else {} - - mtime = pred_path.stat().st_mtime - return { - "tag": tag, - "records": len(records), - "accuracy": metrics.get("accuracy"), - "unparseable": metrics.get("unparseable"), - "total_target": 1730, - "latency_p50_ms": (metrics.get("latency") or {}).get("p50_ms"), - "latency_p90_ms": (metrics.get("latency") or {}).get("p90_ms"), - "file_mtime_epoch": mtime, - } - - -@app.local_entrypoint() -def check(provider: str, model: str, setting: str = "standard", - reasoning: str = "off"): - """Print live progress for a (running or finished) benchmark, non-disruptively.""" - result = _check_remote.remote(provider, model, setting, reasoning) - print(json.dumps(result, indent=2)) - - -# --------------------------------------------------------------------------- -# Download: pull the JSONL + metrics back to ./results/ on the local machine. -# --------------------------------------------------------------------------- - -@app.function( - image=image, - volumes={"/results": results_vol}, - timeout=300, -) -def _list_remote() -> list[str]: - return sorted(p.name for p in Path("/results").iterdir() if p.is_file()) - - -@app.local_entrypoint() -def download(provider: str, model: str, setting: str = "standard", - reasoning: str = "off"): - """Stream the JSONL + metrics for one run back to local ./results/.""" - tag = _tag(provider, model, setting, reasoning) - out_dir = REPO_ROOT / "results" - out_dir.mkdir(parents=True, exist_ok=True) - - files = [f"{tag}_responses.jsonl", f"{tag}_metrics.json"] - for fname in files: - local = out_dir / fname - with local.open("wb") as f: - for chunk in results_vol.read_file(fname): - f.write(chunk) - print(f"wrote {local} ({local.stat().st_size} bytes)") - - -@app.local_entrypoint() -def ls(): - """List files currently in the results volume.""" - files = _list_remote.remote() - for f in files: - print(f) diff --git a/scripts/run_fireworks_suite.sh b/scripts/run_fireworks_suite.sh index 0cbbf81..3cc37a5 100755 --- a/scripts/run_fireworks_suite.sh +++ b/scripts/run_fireworks_suite.sh @@ -76,11 +76,15 @@ fi run_step spider2 uv run -m benchmarks.spider2_lite.spider2_lite \ --provider "$PROVIDER" --model "$MODEL" -run_step mmmupro_standard uv run -m benchmarks.mmmu_pro.mmmu_pro_multi \ - --provider "$PROVIDER" --model "$MODEL" --setting standard --reasoning "$REASONING" - -run_step mmmupro_vision uv run -m benchmarks.mmmu_pro.mmmu_pro_multi \ - --provider "$PROVIDER" --model "$MODEL" --setting vision --reasoning "$REASONING" +if [ -n "${TARGET:-}" ]; then + R="$([ "$REASONING" = "high" ] && echo high || echo off)" + run_step mmmupro_standard uv run python -m bench_core run --benchmark mmmu_pro \ + --target "$TARGET" --variant standard --reasoning "$R" + run_step mmmupro_vision uv run python -m bench_core run --benchmark mmmu_pro \ + --target "$TARGET" --variant vision --reasoning "$R" +else + echo " [skip] mmmupro: set TARGET= (python -m bench_core list-targets)" +fi run_step refcoco uv run -m benchmarks.obj_detection.refcoco_multi \ --provider "$PROVIDER" --model "$MODEL" --split "$REFCOCO_SPLIT" diff --git a/tests/test_mmmu_pro_bench.py b/tests/test_mmmu_pro_bench.py new file mode 100644 index 0000000..ee93950 --- /dev/null +++ b/tests/test_mmmu_pro_bench.py @@ -0,0 +1,62 @@ +"""MMMU-Pro: parse units + offline scoring-parity replay (standard + vision).""" + +import json +import pathlib + +import pytest + +from benchmarks.mmmu_pro import bench + +RESULTS = pathlib.Path(__file__).resolve().parent.parent / "results" +ARCHIVED_TAGS = [ + "mmmupro_standard_fireworks_accounts-fireworks-models-inkling_reasoningoff", + "mmmupro_standard_gemini_gemini-3-7-flash_reasoningoff", + "mmmupro_standard_openrouter_thinkingmachines-inkling-small_reasoningoff", + "mmmupro_vision_fireworks_accounts-fireworks-models-inkling_reasoningoff", + "mmmupro_vision_gemini_gemini-3-7-flash_reasoningoff", + "mmmupro_vision_openrouter_thinkingmachines-inkling-small_reasoningoff", +] + + +def test_parse_answer_a_to_j(): + assert bench.parse_answer("H") == "H" + assert bench.parse_answer("(J)") == "J" + assert bench.parse_answer("The answer is E.") == "E" + assert bench.parse_answer("") is None + + +def test_options_block(): + assert bench._options_block(["x", "y", "z"]) == "A. x\nB. y\nC. z" + + +@pytest.mark.parametrize("tag", ARCHIVED_TAGS) +def test_scoring_parity_with_archived_run(tag): + resp_path = RESULTS / f"{tag}_responses.jsonl" + metrics_path = RESULTS / f"{tag}_metrics.json" + if not (resp_path.exists() and metrics_path.exists()): + pytest.skip(f"archived run {tag} not present") + + archived = json.loads(metrics_path.read_text()) + samples, records = [], [] + for line in resp_path.read_text().splitlines(): + if not line.strip(): + continue + rec = json.loads(line) + if rec.get("response") is None: + continue + samples.append( + { + "id": rec["id"], + "answer": rec["answer"], + "subject": rec.get("subject"), + "topic_difficulty": rec.get("topic_difficulty"), + } + ) + records.append( + {"id": rec["id"], "prediction": bench.parse_answer(rec["response"])} + ) + + m = bench.score(records, samples) + assert m["num_samples"] == archived["num_samples"] + assert m["accuracy"] == pytest.approx(archived["accuracy"]) + assert m["unparseable"] == archived["unparseable"] From 9d03e15638bf147056387d18efe161cae4306ff2 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 08:49:02 +0530 Subject: [PATCH 11/38] feat(bench_core): migrate RefCOCO onto the abstraction; retire 5 forks + oracle re-scorer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit benchmarks/obj_detection/bench.py ports the coordinate parser, IoU scoring, and the format-tolerant oracle (now a score-time submetric, not a separate pass). Retires refcoco_{multi,openai,gemini,grok,kimi}.py and reeval_format_tolerant.py. Offline parity replays strict Acc@0.5 (inkling val 0.3237, inkling-small 0.5987, gemini testA 0.7970) and the oracle (0.8119). Kept: refcoco.py (shared helper lib) and ob_det_api.py — the JigsawStack object-detection benchmark is a separate target, not a per-provider fork, and still imports refcoco.py's helpers. CLI --variant selects the split. 146 tests pass. --- README.md | 21 +- bench_core/cli.py | 1 + benchmarks/obj_detection/bench.py | 285 +++++++++ .../obj_detection/reeval_format_tolerant.py | 220 ------- benchmarks/obj_detection/refcoco_gemini.py | 351 ---------- benchmarks/obj_detection/refcoco_grok.py | 288 --------- benchmarks/obj_detection/refcoco_kimi.py | 290 --------- benchmarks/obj_detection/refcoco_multi.py | 598 ------------------ benchmarks/obj_detection/refcoco_openai.py | 306 --------- scripts/run_fireworks_suite.sh | 13 +- tests/test_refcoco_bench.py | 89 +++ 11 files changed, 390 insertions(+), 2072 deletions(-) create mode 100644 benchmarks/obj_detection/bench.py delete mode 100644 benchmarks/obj_detection/reeval_format_tolerant.py delete mode 100644 benchmarks/obj_detection/refcoco_gemini.py delete mode 100644 benchmarks/obj_detection/refcoco_grok.py delete mode 100644 benchmarks/obj_detection/refcoco_kimi.py delete mode 100644 benchmarks/obj_detection/refcoco_multi.py delete mode 100644 benchmarks/obj_detection/refcoco_openai.py create mode 100644 tests/test_refcoco_bench.py diff --git a/README.md b/README.md index 1a0f223..9535dcc 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ uv run python -m bench_core run --target inkling --benchmark gpqa uv run python -m bench_core run --target inkling --benchmark mmmlu uv run python -m bench_core run --target inkling --benchmark mmmu_pro --variant standard uv run python -m bench_core run --target inkling --benchmark mmmu_pro --variant vision -uv run -m benchmarks.obj_detection.refcoco_multi --provider fireworks --model accounts/fireworks/models/inkling +uv run python -m bench_core run --target inkling --benchmark refcoco --variant val uv run python -m bench_core run --target inkling --benchmark asr uv run -m benchmarks.spider2_lite.spider2_lite --provider fireworks ``` @@ -190,21 +190,14 @@ Links: [RefCOCO/RefCOCO+ paper](https://arxiv.org/abs/1608.00272) · [RefCOCOg p Metric: Acc@IoU=0.5 on the referring-expression bounding box. ```bash -# Interfaze (RefCOCO val by default) -uv run -m benchmarks.obj_detection.refcoco -uv run -m benchmarks.obj_detection.refcoco --split testA -uv run -m benchmarks.obj_detection.refcoco --dataset lmms-lab/RefCOCO+ --split testB +# Any target; --variant is the split (val | testA | testB | test). Metrics include +# a strict Acc@0.5 plus a format-tolerant `oracle` upper bound (submetrics.oracle). +uv run python -m bench_core run --target gpt-5.5 --benchmark refcoco --variant val +uv run python -m bench_core run --target gemini-3.7-flash --benchmark refcoco --variant testA +uv run python -m bench_core run --target inkling --benchmark refcoco --variant val -# Any provider via the multi runner -uv run -m benchmarks.obj_detection.refcoco_multi --provider openai --model gpt-5.4 -uv run -m benchmarks.obj_detection.refcoco_multi --provider anthropic --model claude-sonnet-4-6 -uv run -m benchmarks.obj_detection.refcoco_multi --provider gemini --model gemini-3-flash-preview - -# JigsawStack object_detection API (instead of a VLM) +# JigsawStack object_detection API (a bespoke endpoint, not a chat model) uv run -m benchmarks.obj_detection.ob_det_api - -# Evaluate only -uv run -m benchmarks.obj_detection.refcoco --evaluate-only ``` --- diff --git a/bench_core/cli.py b/bench_core/cli.py index 0b3bfb2..24cd429 100644 --- a/bench_core/cli.py +++ b/bench_core/cli.py @@ -21,6 +21,7 @@ "asr": "benchmarks.asr.bench", "mmmlu": "benchmarks.mmmlu.bench", "mmmu_pro": "benchmarks.mmmu_pro.bench", + "refcoco": "benchmarks.obj_detection.bench", } diff --git a/benchmarks/obj_detection/bench.py b/benchmarks/obj_detection/bench.py new file mode 100644 index 0000000..0c8622d --- /dev/null +++ b/benchmarks/obj_detection/bench.py @@ -0,0 +1,285 @@ +"""RefCOCO grounding: predict one box per referring expression; Acc@IoU=0.5. + +Reports a strict single-interpretation score plus a format-tolerant `oracle` +sub-score that tries {pixel, pixel2x, norm1000, norm1} x {xyxy, yxyx} against GT +(pixel2x covers models like Inkling that report in a 2x-upscaled image space). +""" + +from __future__ import annotations + +import re +from collections import defaultdict + +from bench_core.media import encode_image +from bench_core.request import Message, ReasoningSpec, Request, TextPart + +NAME = "refcoco" +ID_KEY = "id" +PRIMARY_METRIC = "accuracy" +VARIANTS = ["val", "testA", "testB", "test"] +DEFAULTS = {"reasoning": "off", "rate_limit": 25, "max_in_flight": 8} + +_DATASET_ID = "lmms-lab/RefCOCO" +_MAX_SIDE = 1024 +IOU_THRESHOLD = 0.5 +_THRESHOLDS = [0.3, 0.5, 0.7, 0.75, 0.9] + +_PROMPT = ( + "Please provide the bounding box coordinate of the region this sentence describes: " + "{expression}\n\n" + "Output the coordinates in the format [x_min, y_min, x_max, y_max]." +) + +# --- production single-interpretation parser (from refcoco.py) --- +BOX_LINE_PATTERN = re.compile( + r"(?im)^\s*(?:box|answer|bounding\s*box)[\s:]*" + r"\[?\s*(-?\d+(?:\.\d+)?)\s*[,\s]\s*(-?\d+(?:\.\d+)?)\s*[,\s]\s*" + r"(-?\d+(?:\.\d+)?)\s*[,\s]\s*(-?\d+(?:\.\d+)?)\s*\]?" +) +BOXED_PATTERN = re.compile( + r"\\boxed\{\s*\[?\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*" + r"(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\]?\s*\}" +) +BARE_4TUPLE = re.compile( + r"\[\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*" + r"(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\]" +) +JSON_TLBR_PATTERN = re.compile( + r'"top_left"\s*:\s*\{[^}]*?"x"\s*:\s*(-?\d+(?:\.\d+)?)[^}]*?"y"\s*:\s*(-?\d+(?:\.\d+)?)' + r'[^}]*?\}[^}]*?"bottom_right"\s*:\s*\{[^}]*?"x"\s*:\s*(-?\d+(?:\.\d+)?)' + r'[^}]*?"y"\s*:\s*(-?\d+(?:\.\d+)?)', + re.DOTALL, +) +JSON_BOX2D_PATTERN = re.compile( + r'"box_2d"\s*:\s*\[\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*' + r"(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\]" +) + + +def _normalize_to_pixels(box, width, height, order): + max_coord = max(abs(c) for c in box) + if max_coord <= 1.0: + sx, sy = float(width), float(height) + elif max_coord <= 1000 and ( + max(width, height) > 1000 + or box[0] > width + or box[2] > width + or box[1] > height + or box[3] > height + ): + sx, sy = width / 1000.0, height / 1000.0 + else: + sx = sy = 1.0 + if order == "xyxy": + return [box[0] * sx, box[1] * sy, box[2] * sx, box[3] * sy] + return [box[1] * sx, box[0] * sy, box[3] * sx, box[2] * sy] + + +def parse_box(text: str, width: int, height: int): + if not text: + return None + m = None + for match in JSON_TLBR_PATTERN.finditer(text): + m = match + if m is not None: + return _normalize_to_pixels( + [float(m.group(i)) for i in range(1, 5)], width, height, "xyxy" + ) + m = None + for match in JSON_BOX2D_PATTERN.finditer(text): + m = match + if m is not None: + return _normalize_to_pixels( + [float(m.group(i)) for i in range(1, 5)], width, height, "yxyx" + ) + for pattern in (BOX_LINE_PATTERN, BOXED_PATTERN): + matches = list(pattern.finditer(text)) + if matches: + m = matches[-1] + return _normalize_to_pixels( + [float(m.group(i)) for i in range(1, 5)], width, height, "xyxy" + ) + matches = list(BARE_4TUPLE.finditer(text[-800:])) + if matches: + m = matches[-1] + return _normalize_to_pixels( + [float(m.group(i)) for i in range(1, 5)], width, height, "xyxy" + ) + return None + + +def compute_iou(a, b) -> float: + ax1, ay1, ax2, ay2 = a + bx1, by1, bx2, by2 = b + ax1, ax2 = min(ax1, ax2), max(ax1, ax2) + ay1, ay2 = min(ay1, ay2), max(ay1, ay2) + bx1, bx2 = min(bx1, bx2), max(bx1, bx2) + by1, by2 = min(by1, by2), max(by1, by2) + ix1, iy1, ix2, iy2 = max(ax1, bx1), max(ay1, by1), min(ax2, bx2), min(ay2, by2) + if ix1 >= ix2 or iy1 >= iy2: + return 0.0 + inter = (ix2 - ix1) * (iy2 - iy1) + area_a = max(0.0, ax2 - ax1) * max(0.0, ay2 - ay1) + area_b = max(0.0, bx2 - bx1) * max(0.0, by2 - by1) + union = area_a + area_b - inter + return inter / union if union > 0 else 0.0 + + +def coco_bbox_to_xyxy(bbox, sx=1.0, sy=1.0): + x, y, w, h = bbox + return [x * sx, y * sy, (x + w) * sx, (y + h) * sy] + + +# --- format-tolerant oracle (from reeval_format_tolerant.py) --- +_TUPLE = BARE_4TUPLE +_PAREN_PAIRS = re.compile( + r"\(\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\)\s*[^()]*?\s*" + r"\(\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\)" +) + + +def _extract_tuples(text: str): + out = [] + for m in _TUPLE.finditer(text): + out.append(tuple(float(g) for g in m.groups())) + for m in JSON_TLBR_PATTERN.finditer(text): + out.append(tuple(float(g) for g in m.groups())) + for m in _PAREN_PAIRS.finditer(text): + out.append(tuple(float(g) for g in m.groups())) + return out + + +def _all_interpretations(nums, w, h): + n0, n1, n2, n3 = nums + mx = max(abs(c) for c in nums) + yield "pixel-xyxy", [n0, n1, n2, n3] + yield "pixel-yxyx", [n1, n0, n3, n2] + if mx > min(w, h): + yield "pixel2x-xyxy", [n0 / 2, n1 / 2, n2 / 2, n3 / 2] + yield "pixel2x-yxyx", [n1 / 2, n0 / 2, n3 / 2, n2 / 2] + if mx <= 1000: + yield ( + "norm1000-xyxy", + [n0 * w / 1000, n1 * h / 1000, n2 * w / 1000, n3 * h / 1000], + ) + yield ( + "norm1000-yxyx", + [n1 * w / 1000, n0 * h / 1000, n3 * w / 1000, n2 * h / 1000], + ) + if mx <= 1.0: + yield "norm1-xyxy", [n0 * w, n1 * h, n2 * w, n3 * h] + yield "norm1-yxyx", [n1 * w, n0 * h, n3 * w, n2 * h] + + +def _best_box(response: str, w: int, h: int, gt): + best_iou, best_label = 0.0, None + for nums in _extract_tuples(response): + for label, box in _all_interpretations(nums, w, h): + v = compute_iou(box, gt) + if v > best_iou: + best_iou, best_label = v, label + return best_iou, best_label + + +# --- benchmark interface --- +def _sent_dims(w, h, max_side=_MAX_SIDE): + longest = max(w, h) + if longest <= max_side: + return w, h + scale = max_side / longest + return round(w * scale), round(h * scale) + + +def load_samples(sample_size: int | None = None, variant: str = "val") -> list[dict]: + from datasets import load_dataset + + ds = load_dataset(_DATASET_ID, split=variant) + rows = list(ds)[:sample_size] if sample_size else list(ds) + samples = [] + for i, row in enumerate(rows): + answers = row.get("answer") + if isinstance(answers, str): + exprs = [answers] + elif isinstance(answers, list): + exprs = [str(a) for a in answers if str(a).strip()] + else: + exprs = [] + if not exprs: + continue + image = row["image"] + ow, oh = image.size + sw, sh = _sent_dims(ow, oh) + gt = coco_bbox_to_xyxy(list(row["bbox"]), sw / ow, sh / oh) + samples.append( + { + "id": f"{row.get('question_id', i)}_{i}", + "expression": exprs[0], + "image": image, + "sent_w": sw, + "sent_h": sh, + "gt_bbox_xyxy": gt, + } + ) + return samples + + +def build_request(sample: dict, mode: str) -> Request: + prompt = _PROMPT.format(expression=sample["expression"]) + img = encode_image(sample["image"], "image/jpeg", max_side=_MAX_SIDE) + return Request( + [Message("user", [TextPart(prompt), img])], ReasoningSpec(mode), temperature=0.0 + ) + + +def parse(response, sample): + return parse_box(response.text or "", sample["sent_w"], sample["sent_h"]) + + +def _sweep(ious): + n = len(ious) + return { + f"acc@{t}": (sum(1 for i in ious if i >= t) / n if n else 0) + for t in _THRESHOLDS + } + + +def score(records: list[dict], samples: list[dict]) -> dict: + by_id = {s["id"]: s for s in samples} + latest = {r["id"]: r for r in records if r["id"] in by_id} + + strict_ious, oracle_ious = [], [] + strict_correct = oracle_correct = unparsed = total = 0 + labels: dict[str, int] = defaultdict(int) + for rid, r in latest.items(): + s = by_id[rid] + gt, w, h = s["gt_bbox_xyxy"], s["sent_w"], s["sent_h"] + total += 1 + pred = r.get("prediction") + iou = compute_iou(pred, gt) if pred else 0.0 + strict_ious.append(iou) + if pred is not None and iou >= IOU_THRESHOLD: + strict_correct += 1 + if pred is None: + unparsed += 1 + oiou, olabel = _best_box(r.get("response") or "", w, h, gt) + oracle_ious.append(oiou) + if oiou >= IOU_THRESHOLD: + oracle_correct += 1 + if olabel: + labels[olabel] += 1 + + return { + "accuracy": strict_correct / total if total else 0.0, + "correct": strict_correct, + "total": total, + "unparsed": unparsed, + "mean_iou": sum(strict_ious) / len(strict_ious) if strict_ious else 0.0, + "iou_thresholds": _sweep(strict_ious), + "oracle": { + "accuracy": oracle_correct / total if total else 0.0, + "correct": oracle_correct, + "total": total, + "mean_iou": sum(oracle_ious) / len(oracle_ious) if oracle_ious else 0.0, + "interpretation_counts": dict(labels), + }, + } diff --git a/benchmarks/obj_detection/reeval_format_tolerant.py b/benchmarks/obj_detection/reeval_format_tolerant.py deleted file mode 100644 index 8996cf3..0000000 --- a/benchmarks/obj_detection/reeval_format_tolerant.py +++ /dev/null @@ -1,220 +0,0 @@ -""" -Format-tolerant re-evaluation of RefCOCO grounding runs. - -Rationale: under the canonical eval prompt, different models emit boxes in -different conventions — xyxy vs yxyx ordering, 0-1.0 floats / 0-1000 -normalized / raw pixels. The sample-by-sample correct-vs-wrong split under -a single-interpretation parser conflates two very different things: - - 1. Did the model find the right region? (grounding capability — what we - actually want to measure) - 2. Did the model emit it in the format we happened to parse? (format - compliance — a separate, prompt-dependent concern) - -This script re-scores existing `*_responses.jsonl` files by, for each -sample, trying every reasonable interpretation of every 4-tuple in the -response and keeping the one whose IoU with GT is highest. That isolates -(1) by taking (2) out of the equation — applied uniformly to every model -so the comparison stays apples-to-apples. - -This is oracle parsing — it uses GT to disambiguate, so the resulting -numbers are *upper bounds* on production performance. Single-interpretation -numbers stay in the originals. - -Usage: - uv run -m benchmarks.obj_detection.reeval_format_tolerant -""" - -import json -import re -import sys -from pathlib import Path - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -from benchmarks.obj_detection.refcoco import ( # noqa: E402 - compute_iou, - compute_metrics, -) - -RESULTS_DIR = PROJECT_ROOT / "results" -TUPLE_PATTERN = re.compile( - r"\[\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*" - r"(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\]" -) -PAREN_PAIRS = re.compile( - r"\(\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\)\s*[^()]*?\s*" - r"\(\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\)" -) -TLBR_JSON = re.compile( - r'"top_left"\s*:\s*\{[^}]*?"x"\s*:\s*(-?\d+(?:\.\d+)?)[^}]*?"y"\s*:\s*' - r'(-?\d+(?:\.\d+)?)[^}]*?\}[^}]*?"bottom_right"\s*:\s*\{[^}]*?"x"\s*:\s*' - r'(-?\d+(?:\.\d+)?)[^}]*?"y"\s*:\s*(-?\d+(?:\.\d+)?)', - re.DOTALL, -) - - -def extract_tuples(text: str) -> list[tuple[float, float, float, float]]: - """Pull every plausible 4-tuple out of the response — bracket lists, - parenthesized (x,y) pairs, and JSON top_left/bottom_right blocks. - Order is preserved (so callers can prefer the last match if needed).""" - out = [] - for m in TUPLE_PATTERN.finditer(text): - out.append(tuple(float(g) for g in m.groups())) - for m in TLBR_JSON.finditer(text): - # already xyxy in pixel space — emit as-is. - out.append(tuple(float(g) for g in m.groups())) - for m in PAREN_PAIRS.finditer(text): - out.append(tuple(float(g) for g in m.groups())) - return out - - -def all_interpretations(nums, w, h): - """Yield (label, [x1,y1,x2,y2]) for every plausible interpretation - of a 4-tuple under the canonical RefCOCO eval — covers xyxy/yxyx - order × {raw pixel, 2x-upscaled pixel, 0-1000 normalized, 0-1.0 float} - scale. - - The 2x-pixel family covers models that report coordinates in their own - internally-upscaled image space rather than the sent image's — Inkling - does this for images below its vision encoder's working resolution - (verified: a box at 50-75% of a 640x428 input comes back as - [500, 210, 750, 430], i.e. that region in 1280x856 space). Applied to - every model uniformly, like every other hypothesis here.""" - n0, n1, n2, n3 = nums - mx = max(abs(c) for c in nums) - yield "pixel-xyxy", [n0, n1, n2, n3] - yield "pixel-yxyx", [n1, n0, n3, n2] - # Only worth trying when the tuple overflows the image — otherwise it is - # just a shrunken duplicate of the pixel hypothesis. - if mx > min(w, h): - yield "pixel2x-xyxy", [n0 / 2, n1 / 2, n2 / 2, n3 / 2] - yield "pixel2x-yxyx", [n1 / 2, n0 / 2, n3 / 2, n2 / 2] - if mx <= 1000: - yield ( - "norm1000-xyxy", - [n0 * w / 1000, n1 * h / 1000, n2 * w / 1000, n3 * h / 1000], - ) - yield ( - "norm1000-yxyx", - [n1 * w / 1000, n0 * h / 1000, n3 * w / 1000, n2 * h / 1000], - ) - if mx <= 1.0: - yield "norm1-xyxy", [n0 * w, n1 * h, n2 * w, n3 * h] - yield "norm1-yxyx", [n1 * w, n0 * h, n3 * w, n2 * h] - - -def best_box(response: str, w: int, h: int, gt: list[float]): - """Return (best_box_xyxy, best_iou, label) — picks the interpretation - of any 4-tuple in the response that maximizes IoU vs GT.""" - best_iou = 0.0 - best_box = None - best_label = None - for nums in extract_tuples(response): - for label, box in all_interpretations(nums, w, h): - v = compute_iou(box, gt) - if v > best_iou: - best_iou, best_box, best_label = v, box, label - return best_box, best_iou, best_label - - -def reeval_file(path: Path) -> tuple[list[dict], dict]: - records = [] - label_counts = {} - n_changed = 0 - with open(path) as f: - for line in f: - line = line.strip() - if not line: - continue - r = json.loads(line) - gt = r["gt_bbox_xyxy"] - w = r["image_width"] - h = r["image_height"] - box, iou, label = best_box(r["response"], w, h, gt) - new_correct = box is not None and iou >= 0.5 - if r.get("correct") != new_correct: - n_changed += 1 - r["pred_bbox_xyxy"] = box - r["iou"] = iou - r["correct"] = new_correct - r["interpretation"] = label - records.append(r) - if box is not None: - label_counts[label] = label_counts.get(label, 0) + 1 - return records, {"n_changed": n_changed, "label_counts": label_counts} - - -def write_records(records: list[dict], path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w") as f: - for r in records: - f.write(json.dumps(r, ensure_ascii=False) + "\n") - - -def discover_targets() -> list[tuple[str, str]]: - """Every RefCOCO response file in results/ — identified by the grounding - fields rather than a filename convention, since the per-provider runners - tag their outputs differently. Skips this script's own oracle outputs.""" - targets = [] - for path in sorted(RESULTS_DIR.glob("*_responses.jsonl")): - if "_oracle_" in path.name: - continue - try: - with open(path) as f: - first = json.loads(f.readline() or "{}") - except (json.JSONDecodeError, OSError): - continue - if "gt_bbox_xyxy" not in first: - continue - tag = path.name[: -len("_responses.jsonl")] - label = first.get("model") or first.get("provider") or tag - targets.append((label, tag)) - return targets - - -def main(): - targets = discover_targets() - if not targets: - print(f"No RefCOCO response files found in {RESULTS_DIR}") - return - print( - f"{'model':<14} {'orig_acc':>10} {'oracle_acc':>11} {'mean_iou':>10} {'changed':>9}" - ) - print("-" * 60) - for label, tag in targets: - src = RESULTS_DIR / f"{tag}_responses.jsonl" - if not src.exists(): - print(f"{label:<14} (missing {src.name})") - continue - # Original score (before re-eval) - with open(src) as f: - orig_records = [json.loads(l) for l in f if l.strip()] - orig_correct = sum(1 for r in orig_records if r.get("correct")) - orig_total = len(orig_records) - - records, info = reeval_file(src) - out_responses = RESULTS_DIR / f"{tag}_oracle_responses.jsonl" - out_metrics = RESULTS_DIR / f"{tag}_oracle_metrics.json" - write_records(records, out_responses) - - metrics = compute_metrics(records) - metrics["interpretation_counts"] = info["label_counts"] - metrics["records_changed_from_orig"] = info["n_changed"] - metrics["model"] = label - with open(out_metrics, "w") as f: - json.dump(metrics, f, indent=2) - - print( - f"{label:<14} {orig_correct / orig_total:>10.4f} " - f"{metrics['accuracy']:>11.4f} {metrics['mean_iou']:>10.4f} " - f"{info['n_changed']:>9d}" - ) - print( - f" interpretations used: {dict(sorted(info['label_counts'].items(), key=lambda x: -x[1]))}" - ) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/obj_detection/refcoco_gemini.py b/benchmarks/obj_detection/refcoco_gemini.py deleted file mode 100644 index cf55bb7..0000000 --- a/benchmarks/obj_detection/refcoco_gemini.py +++ /dev/null @@ -1,351 +0,0 @@ -""" -RefCOCO benchmark for Gemini 2.5 Pro. - -Mirrors the protocol used by `refcoco.py` (interfaze-beta) so numbers are -directly comparable on the same splits at Acc@IoU=0.5. Only the inference -call changes — same prompt, same parser, same IoU, same checkpointing. - -Reasoning: Gemini 2.5 Pro cannot fully disable thinking (min budget = 128). -We use thinking_budget=128 to approximate the interfaze "reasoning off" -default. Temperature pinned to 0.0 to match. - -Datasets (lmms-lab packaging — same as standard RefCOCO splits): - lmms-lab/RefCOCO val 8811, test 5000, testA 1975, testB 1810 - lmms-lab/RefCOCO+ val 8823, testA 1975, testB 1798 - lmms-lab/RefCOCOg val 7573, test 9602 (UMD split) - -Usage: - uv run -m benchmarks.obj_detection.refcoco_gemini --split testA - uv run -m benchmarks.obj_detection.refcoco_gemini --split testA --limit 20 - uv run -m benchmarks.obj_detection.refcoco_gemini --split testA --evaluate-only - -Env: GEMINI_KEY must be set (loaded from .env). -""" - -import sys -import json -import os -import re -import time -import asyncio -import argparse -import traceback -import concurrent.futures -from pathlib import Path - -from datasets import load_dataset -from dotenv import load_dotenv -from google import genai -from google.genai import types -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -from benchmarks.obj_detection.refcoco import ( # noqa: E402 - JsonlWriter, - PROMPT_TEMPLATE, - build_samples, - coco_bbox_to_xyxy, - compute_iou, - compute_metrics, - load_completed_ids, - load_records, - parse_box, - pil_to_data_url, -) - -load_dotenv() - -RESULTS_DIR = PROJECT_ROOT / "results" -DEFAULT_DATASET = "lmms-lab/RefCOCO" -DEFAULT_SPLIT = "val" -MODEL = "gemini-2.5-pro" -THINKING_BUDGET = 128 # Pro min — Pro API rejects 0 ("only works in thinking mode") -TEMPERATURE = 0.0 -CONCURRENCY = 5 # Pro 503s ("model overloaded") under heavier load -MAX_RETRIES = 6 -RETRY_BACKOFF_CAP_S = 30.0 -IOU_THRESHOLD = 0.5 - -GEMINI_KEY = os.getenv("GEMINI_KEY") or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY") -if not GEMINI_KEY: - raise RuntimeError( - "GEMINI_KEY is not set. Add it to .env " - "(get one from https://aistudio.google.com/app/apikey)." - ) - -# google-genai's Client is thread-safe; one shared instance is fine. -gemini_client = genai.Client(api_key=GEMINI_KEY) - - -def invoke_gemini(prompt: str, jpeg_bytes: bytes): - """Single Gemini 2.5 Pro generate_content call. Returns the SDK response.""" - contents = [ - types.Content( - role="user", - parts=[ - types.Part.from_text(text=prompt), - types.Part.from_bytes(data=jpeg_bytes, mime_type="image/jpeg"), - ], - ), - ] - config = types.GenerateContentConfig( - temperature=TEMPERATURE, - thinking_config=types.ThinkingConfig(thinking_budget=THINKING_BUDGET), - ) - return gemini_client.models.generate_content( - model=MODEL, - contents=contents, - config=config, - ) - - -def data_url_to_jpeg_bytes(data_url: str) -> bytes: - """Strip the `data:image/jpeg;base64,` prefix and decode.""" - import base64 - _, b64 = data_url.split(",", 1) - return base64.b64decode(b64) - - -# Gemini's native bounding-box convention: [ymin, xmin, ymax, xmax] normalized -# to 0-1000 — documented in https://ai.google.dev/gemini-api/docs/image-understanding -# (and the format every Gemini grounding example uses). Asking for this format -# is the fair head-to-head equivalent of the Qwen / InternVL / interfaze prompts: -# each model is queried in the coordinate convention it was trained on. -PROMPT_GEMINI_TEMPLATE = """Locate the single region in the image described by the following expression and return its bounding box. - -Expression: "{expression}" - -Output ONLY a JSON object on the last line in this exact format: -{{"box_2d": [ymin, xmin, ymax, xmax]}} - -Coordinates must be normalized to 0-1000 where (0,0) is the top-left corner and (1000,1000) is the bottom-right corner of the image. Think step by step before answering.""" - - -JSON_BOX_PATTERN = re.compile( - r'"box_2d"\s*:\s*\[\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*' - r'(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\]' -) -BARE_4TUPLE_PATTERN = re.compile( - r'\[\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*' - r'(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\]' -) - - -def parse_box_gemini(text: str, width: int, height: int) -> list[float] | None: - """Parse Gemini's native [ymin, xmin, ymax, xmax] in 0-1000 → pixel xyxy. - - Tries `"box_2d": [...]` first, falls back to the last bare 4-tuple in the - response. Returns [x1, y1, x2, y2] in pixel space, or None if unparseable. - """ - if not text: - return None - m = None - for match in JSON_BOX_PATTERN.finditer(text): - m = match - if m is None: - for match in BARE_4TUPLE_PATTERN.finditer(text): - m = match - if m is None: - return None - ymin, xmin, ymax, xmax = (float(m.group(i)) for i in range(1, 5)) - # Gemini almost always emits 0-1000 normalized; guard against the rare - # case where it emits raw pixels by checking whether values fit in [0, 1000]. - max_coord = max(abs(ymin), abs(xmin), abs(ymax), abs(xmax)) - if max_coord <= 1000: - x1 = xmin * width / 1000.0 - y1 = ymin * height / 1000.0 - x2 = xmax * width / 1000.0 - y2 = ymax * height / 1000.0 - else: - x1, y1, x2, y2 = xmin, ymin, xmax, ymax - return [x1, y1, x2, y2] - - -async def process_sample(sample: dict, semaphore: asyncio.Semaphore, - writer: JsonlWriter, progress: dict) -> dict | None: - orig_w, orig_h = sample["image"].size - data_url, sent_w, sent_h = pil_to_data_url(sample["image"]) - jpeg_bytes = data_url_to_jpeg_bytes(data_url) - sx = sent_w / orig_w - sy = sent_h / orig_h - gt_xyxy = coco_bbox_to_xyxy(sample["bbox_xywh"], sx, sy) - - prompt = PROMPT_TEMPLATE.format( - width=sent_w, height=sent_h, expression=sample["expression"] - ) - last_error: str | None = None - - for attempt in range(1, MAX_RETRIES + 1): - start = time.perf_counter() - try: - async with semaphore: - start = time.perf_counter() - response = await asyncio.to_thread(invoke_gemini, prompt, jpeg_bytes) - latency_ms = int((time.perf_counter() - start) * 1000) - content = (response.text or "").strip() - request_id = getattr(response, "response_id", None) - if not content: - last_error = "empty response content" - raise RuntimeError(last_error) - - pred_box = parse_box(content, sent_w, sent_h) - iou = compute_iou(pred_box, gt_xyxy) if pred_box else 0.0 - correct = pred_box is not None and iou >= IOU_THRESHOLD - - record = { - "id": sample["id"], - "question_id": sample["question_id"], - "file_name": sample["file_name"], - "expression": sample["expression"], - "all_expressions": sample["all_expressions"], - "image_width": sent_w, - "image_height": sent_h, - "gt_bbox_xyxy": gt_xyxy, - "pred_bbox_xyxy": pred_box, - "iou": iou, - "correct": correct, - "response": content, - "request_id": request_id, - "latency_ms": latency_ms, - "attempts": attempt, - } - await writer.append(record) - - progress["done"] += 1 - if correct: - progress["correct"] += 1 - mark = "OK" - else: - mark = "X " - tqdm.write( - f"[{progress['done']}/{progress['total']}] {mark} " - f"id={sample['id']} iou={iou:.3f} " - f"pred={pred_box} gt={[round(x,1) for x in gt_xyxy]} " - f"latency={latency_ms}ms req_id={request_id} attempt={attempt}" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - tqdm.write( - f"[error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(min(2 ** (attempt - 1), RETRY_BACKOFF_CAP_S)) - - progress["failed"] += 1 - tqdm.write(f"[FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") - return None - - -def print_summary(metrics: dict, dataset_name: str, split: str): - print(f"\n{'=' * 60}") - print(f"Grounding Results — {dataset_name} / {split} ({MODEL}, thinking_budget={THINKING_BUDGET})") - print(f"{'=' * 60}") - print(f"Acc@IoU=0.5 : {metrics['accuracy']:.4f} ({metrics['correct']}/{metrics['total']})") - print(f"Mean IoU : {metrics['mean_iou']:.4f}") - print(f"Unparsed : {metrics['unparsed']}") - if metrics.get("latency"): - lat = metrics["latency"] - print(f"Latency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " - f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms max={lat['max_ms']}ms") - print("\nIoU thresholds:") - for t, acc in metrics["iou_thresholds"].items(): - print(f" {t}: {acc:.4f}") - - -def build_tag(dataset: str, split: str) -> str: - ds_slug = dataset.split("/")[-1].lower().replace("+", "plus") - return f"gemini25pro_{ds_slug}_{split}" - - -async def run_predictions(dataset_name: str, split: str, pred_path: Path, limit: int | None): - print(f"Loading {dataset_name}, split={split}...") - dataset = load_dataset(dataset_name, split=split) - print(f"Loaded {len(dataset)} rows") - - samples = build_samples(dataset) - done_ids = load_completed_ids(pred_path) - pending = [s for s in samples if s["id"] not in done_ids] - if limit is not None: - pending = pending[:limit] - print(f"--limit applied: will run at most {limit} sample(s)") - print(f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})") - if not pending: - return - - asyncio.get_running_loop().set_default_executor( - concurrent.futures.ThreadPoolExecutor(max_workers=CONCURRENCY) - ) - - writer = JsonlWriter(pred_path) - semaphore = asyncio.Semaphore(CONCURRENCY) - progress = {"total": len(pending), "done": 0, "correct": 0, "failed": 0} - tasks = [process_sample(s, semaphore, writer, progress) for s in pending] - try: - await tqdm_asyncio.gather(*tasks, desc=f"{dataset_name}/{split}") - except Exception: - traceback.print_exc() - print(f"\nRun finished: {progress['done']}/{progress['total']} answered, " - f"{progress['correct']} correct, {progress['failed']} failed.") - - -def run_evaluation(dataset_name: str, split: str, pred_path: Path, metrics_path: Path): - if not pred_path.exists(): - print(f"No predictions found at {pred_path}") - sys.exit(1) - results = load_records(pred_path) - if not results: - print(f"No records in {pred_path}") - sys.exit(1) - - metrics = compute_metrics(results) - print_summary(metrics, dataset_name, split) - output = { - **metrics, - "dataset": dataset_name, "split": split, - "thinking_budget": THINKING_BUDGET, "temperature": TEMPERATURE, - "concurrency": CONCURRENCY, "iou_threshold": IOU_THRESHOLD, "model": MODEL, - } - metrics_path.parent.mkdir(parents=True, exist_ok=True) - with open(metrics_path, "w") as f: - json.dump(output, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - parser = argparse.ArgumentParser( - description="RefCOCO / RefCOCO+ / RefCOCOg benchmark for Gemini 2.5 Pro " - "(Acc@IoU=0.5, comparable to refcoco.py interfaze run)" - ) - parser.add_argument("--dataset", default=DEFAULT_DATASET, - help="lmms-lab/RefCOCO | lmms-lab/RefCOCO+ | lmms-lab/RefCOCOg") - parser.add_argument("--split", default=DEFAULT_SPLIT, - help="val | testA | testB | test (availability varies by dataset)") - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument("--limit", type=int, default=None, - help="Only run the first N unanswered samples") - args = parser.parse_args() - - tag = build_tag(args.dataset, args.split) - pred_path = RESULTS_DIR / f"{tag}_responses.jsonl" - metrics_path = RESULTS_DIR / f"{tag}_metrics.json" - - if args.evaluate_only: - run_evaluation(args.dataset, args.split, pred_path, metrics_path) - elif args.predict_only: - asyncio.run(run_predictions(args.dataset, args.split, pred_path, limit=args.limit)) - else: - asyncio.run(run_predictions(args.dataset, args.split, pred_path, limit=args.limit)) - run_evaluation(args.dataset, args.split, pred_path, metrics_path) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/obj_detection/refcoco_grok.py b/benchmarks/obj_detection/refcoco_grok.py deleted file mode 100644 index 6973c06..0000000 --- a/benchmarks/obj_detection/refcoco_grok.py +++ /dev/null @@ -1,288 +0,0 @@ -""" -RefCOCO benchmark for xAI Grok 4.3 via OpenRouter. - -Strict head-to-head with `refcoco.py` (interfaze): same prompt, same parser, -same IoU, same checkpointing — only the inference call changes. Mirrors -`refcoco_kimi.py` but routed at xAI's Grok 4.3. - -Reasoning: Grok 4.3's OpenRouter endpoint rejects `reasoning.enabled=false` -("Reasoning is mandatory for this endpoint"), so the closest analog to the -other refcoco runs' "reasoning off" is the lowest accepted effort tier, -`reasoning.effort="minimal"`. The reasoning_effort knob is exposed via ---reasoning-effort. Temperature pinned to 0.0. - -Usage: - uv run -m benchmarks.obj_detection.refcoco_grok --split testA - uv run -m benchmarks.obj_detection.refcoco_grok --split testA --limit 20 - uv run -m benchmarks.obj_detection.refcoco_grok --split testA --evaluate-only - -Env: OPENROUTER_API_KEY must be set (loaded from .env). -""" - -import sys -import json -import os -import time -import asyncio -import argparse -import traceback -import concurrent.futures -from pathlib import Path - -from datasets import load_dataset -from dotenv import load_dotenv -from openai import OpenAI -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -from benchmarks.obj_detection.refcoco import ( # noqa: E402 - JsonlWriter, - PROMPT_TEMPLATE, - build_samples, - coco_bbox_to_xyxy, - compute_iou, - compute_metrics, - load_completed_ids, - load_records, - parse_box, - pil_to_data_url, -) - -load_dotenv() - -RESULTS_DIR = PROJECT_ROOT / "results" -DEFAULT_DATASET = "lmms-lab/RefCOCO" -DEFAULT_SPLIT = "val" -MODEL = "x-ai/grok-4.3" -DEFAULT_REASONING_EFFORT = "minimal" -TEMPERATURE = 0.0 -CONCURRENCY = 10 -MAX_RETRIES = 5 -RETRY_BACKOFF_CAP_S = 30.0 -IOU_THRESHOLD = 0.5 - -REASONING_EFFORT = DEFAULT_REASONING_EFFORT - -OPENROUTER_KEY = os.getenv("OPENROUTER_API_KEY") or os.getenv("OPENROUTER_KEY") -if not OPENROUTER_KEY: - raise RuntimeError( - "OPENROUTER_API_KEY is not set. Add it to .env " - "(get one from https://openrouter.ai/keys)." - ) - -openrouter_client = OpenAI( - base_url="https://openrouter.ai/api/v1", - api_key=OPENROUTER_KEY, -) - -def invoke_grok(messages: list[dict]): - """Single Grok 4.3 chat.completions call via OpenRouter.""" - return openrouter_client.chat.completions.create( - model=MODEL, - messages=messages, - temperature=TEMPERATURE, - extra_body={"reasoning": {"effort": REASONING_EFFORT}}, - ) - - -async def process_sample(sample: dict, semaphore: asyncio.Semaphore, - writer: JsonlWriter, progress: dict) -> dict | None: - orig_w, orig_h = sample["image"].size - data_url, sent_w, sent_h = pil_to_data_url(sample["image"]) - sx = sent_w / orig_w - sy = sent_h / orig_h - gt_xyxy = coco_bbox_to_xyxy(sample["bbox_xywh"], sx, sy) - - prompt = PROMPT_TEMPLATE.format( - width=sent_w, height=sent_h, expression=sample["expression"] - ) - messages = [{ - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - {"type": "image_url", "image_url": {"url": data_url}}, - ], - }] - last_error: str | None = None - - for attempt in range(1, MAX_RETRIES + 1): - start = time.perf_counter() - try: - async with semaphore: - start = time.perf_counter() - response = await asyncio.to_thread(invoke_grok, messages) - latency_ms = int((time.perf_counter() - start) * 1000) - content = (response.choices[0].message.content or "").strip() - request_id = getattr(response, "id", None) - if not content: - last_error = "empty response content" - raise RuntimeError(last_error) - - pred_box = parse_box(content, sent_w, sent_h) - iou = compute_iou(pred_box, gt_xyxy) if pred_box else 0.0 - correct = pred_box is not None and iou >= IOU_THRESHOLD - - record = { - "id": sample["id"], - "question_id": sample["question_id"], - "file_name": sample["file_name"], - "expression": sample["expression"], - "all_expressions": sample["all_expressions"], - "image_width": sent_w, - "image_height": sent_h, - "gt_bbox_xyxy": gt_xyxy, - "pred_bbox_xyxy": pred_box, - "iou": iou, - "correct": correct, - "response": content, - "request_id": request_id, - "latency_ms": latency_ms, - "attempts": attempt, - } - await writer.append(record) - - progress["done"] += 1 - if correct: - progress["correct"] += 1 - mark = "OK" - else: - mark = "X " - tqdm.write( - f"[{progress['done']}/{progress['total']}] {mark} " - f"id={sample['id']} iou={iou:.3f} " - f"pred={pred_box} gt={[round(x,1) for x in gt_xyxy]} " - f"latency={latency_ms}ms req_id={request_id} attempt={attempt}" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - tqdm.write( - f"[error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(min(2 ** (attempt - 1), RETRY_BACKOFF_CAP_S)) - - progress["failed"] += 1 - tqdm.write(f"[FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") - return None - - -def print_summary(metrics: dict, dataset_name: str, split: str): - print(f"\n{'=' * 60}") - print(f"Grounding Results — {dataset_name} / {split} ({MODEL}, reasoning_effort={REASONING_EFFORT})") - print(f"{'=' * 60}") - print(f"Acc@IoU=0.5 : {metrics['accuracy']:.4f} ({metrics['correct']}/{metrics['total']})") - print(f"Mean IoU : {metrics['mean_iou']:.4f}") - print(f"Unparsed : {metrics['unparsed']}") - if metrics.get("latency"): - lat = metrics["latency"] - print(f"Latency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " - f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms max={lat['max_ms']}ms") - print("\nIoU thresholds:") - for t, acc in metrics["iou_thresholds"].items(): - print(f" {t}: {acc:.4f}") - - -def build_tag(dataset: str, split: str) -> str: - ds_slug = dataset.split("/")[-1].lower().replace("+", "plus") - return f"grok43_reasoning{REASONING_EFFORT}_{ds_slug}_{split}" - - -async def run_predictions(dataset_name: str, split: str, pred_path: Path, limit: int | None): - print(f"Loading {dataset_name}, split={split}...") - dataset = load_dataset(dataset_name, split=split) - print(f"Loaded {len(dataset)} rows") - - samples = build_samples(dataset) - done_ids = load_completed_ids(pred_path) - pending = [s for s in samples if s["id"] not in done_ids] - if limit is not None: - pending = pending[:limit] - print(f"--limit applied: will run at most {limit} sample(s)") - print(f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})") - if not pending: - return - - asyncio.get_running_loop().set_default_executor( - concurrent.futures.ThreadPoolExecutor(max_workers=CONCURRENCY) - ) - - writer = JsonlWriter(pred_path) - semaphore = asyncio.Semaphore(CONCURRENCY) - progress = {"total": len(pending), "done": 0, "correct": 0, "failed": 0} - tasks = [process_sample(s, semaphore, writer, progress) for s in pending] - try: - await tqdm_asyncio.gather(*tasks, desc=f"{dataset_name}/{split}") - except Exception: - traceback.print_exc() - print(f"\nRun finished: {progress['done']}/{progress['total']} answered, " - f"{progress['correct']} correct, {progress['failed']} failed.") - - -def run_evaluation(dataset_name: str, split: str, pred_path: Path, metrics_path: Path): - if not pred_path.exists(): - print(f"No predictions found at {pred_path}") - sys.exit(1) - results = load_records(pred_path) - if not results: - print(f"No records in {pred_path}") - sys.exit(1) - - metrics = compute_metrics(results) - print_summary(metrics, dataset_name, split) - output = { - **metrics, - "dataset": dataset_name, "split": split, - "reasoning_effort": REASONING_EFFORT, "temperature": TEMPERATURE, - "concurrency": CONCURRENCY, "iou_threshold": IOU_THRESHOLD, - "model": MODEL, "provider": "openrouter", - } - metrics_path.parent.mkdir(parents=True, exist_ok=True) - with open(metrics_path, "w") as f: - json.dump(output, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - global REASONING_EFFORT - parser = argparse.ArgumentParser( - description="RefCOCO benchmark for xAI Grok 4.3 via OpenRouter " - "(strict head-to-head with refcoco.py — same prompt, parser, IoU)" - ) - parser.add_argument("--dataset", default=DEFAULT_DATASET, - help="lmms-lab/RefCOCO | lmms-lab/RefCOCO+ | lmms-lab/RefCOCOg") - parser.add_argument("--split", default=DEFAULT_SPLIT, - help="val | testA | testB | test (availability varies by dataset)") - parser.add_argument("--reasoning-effort", default=DEFAULT_REASONING_EFFORT, - choices=["minimal", "low", "medium", "high"], - help="OpenRouter reasoning.effort (Grok 4.3 cannot fully disable; " - "minimal is the lowest accepted tier)") - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument("--limit", type=int, default=None, - help="Only run the first N unanswered samples") - args = parser.parse_args() - REASONING_EFFORT = args.reasoning_effort - - tag = build_tag(args.dataset, args.split) - pred_path = RESULTS_DIR / f"{tag}_responses.jsonl" - metrics_path = RESULTS_DIR / f"{tag}_metrics.json" - - if args.evaluate_only: - run_evaluation(args.dataset, args.split, pred_path, metrics_path) - elif args.predict_only: - asyncio.run(run_predictions(args.dataset, args.split, pred_path, limit=args.limit)) - else: - asyncio.run(run_predictions(args.dataset, args.split, pred_path, limit=args.limit)) - run_evaluation(args.dataset, args.split, pred_path, metrics_path) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/obj_detection/refcoco_kimi.py b/benchmarks/obj_detection/refcoco_kimi.py deleted file mode 100644 index 9dd4f61..0000000 --- a/benchmarks/obj_detection/refcoco_kimi.py +++ /dev/null @@ -1,290 +0,0 @@ -""" -RefCOCO benchmark for Kimi K2.6 via OpenRouter (provider pinned to Moonshot). - -Strict head-to-head with `refcoco.py` (interfaze): same prompt, same parser, -same IoU, same checkpointing — only the inference call changes. - -Model: `moonshotai/kimi-k2.6` (Apr 20, 2026 release — Moonshot's flagship -multimodal MoE, 1T total / 32B active, 256K context). OpenRouter exposes an -OpenAI-compatible API; we pin provider to `moonshotai` so Moonshot's own -inference is used (not a third-party reseller). - -Reasoning: passed via OpenRouter's `reasoning` extra_body parameter -({"enabled": false}) — equivalent to interfaze's `reasoning_effort=None`. -Temperature pinned to 0.0. - -Usage: - uv run -m benchmarks.obj_detection.refcoco_kimi --split testA - uv run -m benchmarks.obj_detection.refcoco_kimi --split testA --limit 20 - uv run -m benchmarks.obj_detection.refcoco_kimi --split testA --evaluate-only - -Env: OPENROUTER_KEY must be set (loaded from .env). -""" - -import sys -import json -import os -import time -import asyncio -import argparse -import traceback -import concurrent.futures -from pathlib import Path - -from datasets import load_dataset -from dotenv import load_dotenv -from openai import OpenAI -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -from benchmarks.obj_detection.refcoco import ( # noqa: E402 - JsonlWriter, - PROMPT_TEMPLATE, - build_samples, - coco_bbox_to_xyxy, - compute_iou, - compute_metrics, - load_completed_ids, - load_records, - parse_box, - pil_to_data_url, -) - -load_dotenv() - -RESULTS_DIR = PROJECT_ROOT / "results" -DEFAULT_DATASET = "lmms-lab/RefCOCO" -DEFAULT_SPLIT = "val" -MODEL = "moonshotai/kimi-k2.6" -TEMPERATURE = 0.0 -CONCURRENCY = 10 -MAX_RETRIES = 5 -RETRY_BACKOFF_CAP_S = 30.0 -IOU_THRESHOLD = 0.5 - -OPENROUTER_KEY = os.getenv("OPENROUTER_KEY") or os.getenv("OPENROUTER_API_KEY") -if not OPENROUTER_KEY: - raise RuntimeError( - "OPENROUTER_KEY is not set. Add it to .env " - "(get one from https://openrouter.ai/keys)." - ) - -openrouter_client = OpenAI( - base_url="https://openrouter.ai/api/v1", - api_key=OPENROUTER_KEY, -) - -# `reasoning.enabled=false` = thinking off (OpenRouter unified reasoning API). -# `provider.only=["moonshotai"]` pins routing to Moonshot's own inference so -# we're benchmarking Moonshot's deployment, not a downstream reseller's. -EXTRA_BODY = { - "reasoning": {"enabled": False}, - "provider": {"only": ["moonshotai"]}, -} - - -def invoke_kimi(messages: list[dict]): - """Single Kimi K2.6 chat.completions call via OpenRouter.""" - return openrouter_client.chat.completions.create( - model=MODEL, - messages=messages, - temperature=TEMPERATURE, - extra_body=EXTRA_BODY, - ) - - -async def process_sample(sample: dict, semaphore: asyncio.Semaphore, - writer: JsonlWriter, progress: dict) -> dict | None: - orig_w, orig_h = sample["image"].size - data_url, sent_w, sent_h = pil_to_data_url(sample["image"]) - sx = sent_w / orig_w - sy = sent_h / orig_h - gt_xyxy = coco_bbox_to_xyxy(sample["bbox_xywh"], sx, sy) - - prompt = PROMPT_TEMPLATE.format( - width=sent_w, height=sent_h, expression=sample["expression"] - ) - messages = [{ - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - {"type": "image_url", "image_url": {"url": data_url}}, - ], - }] - last_error: str | None = None - - for attempt in range(1, MAX_RETRIES + 1): - start = time.perf_counter() - try: - async with semaphore: - start = time.perf_counter() - response = await asyncio.to_thread(invoke_kimi, messages) - latency_ms = int((time.perf_counter() - start) * 1000) - content = (response.choices[0].message.content or "").strip() - request_id = getattr(response, "id", None) - if not content: - last_error = "empty response content" - raise RuntimeError(last_error) - - pred_box = parse_box(content, sent_w, sent_h) - iou = compute_iou(pred_box, gt_xyxy) if pred_box else 0.0 - correct = pred_box is not None and iou >= IOU_THRESHOLD - - record = { - "id": sample["id"], - "question_id": sample["question_id"], - "file_name": sample["file_name"], - "expression": sample["expression"], - "all_expressions": sample["all_expressions"], - "image_width": sent_w, - "image_height": sent_h, - "gt_bbox_xyxy": gt_xyxy, - "pred_bbox_xyxy": pred_box, - "iou": iou, - "correct": correct, - "response": content, - "request_id": request_id, - "latency_ms": latency_ms, - "attempts": attempt, - } - await writer.append(record) - - progress["done"] += 1 - if correct: - progress["correct"] += 1 - mark = "OK" - else: - mark = "X " - tqdm.write( - f"[{progress['done']}/{progress['total']}] {mark} " - f"id={sample['id']} iou={iou:.3f} " - f"pred={pred_box} gt={[round(x,1) for x in gt_xyxy]} " - f"latency={latency_ms}ms req_id={request_id} attempt={attempt}" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - tqdm.write( - f"[error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(min(2 ** (attempt - 1), RETRY_BACKOFF_CAP_S)) - - progress["failed"] += 1 - tqdm.write(f"[FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") - return None - - -def print_summary(metrics: dict, dataset_name: str, split: str): - print(f"\n{'=' * 60}") - print(f"Grounding Results — {dataset_name} / {split} ({MODEL}, reasoning=off)") - print(f"{'=' * 60}") - print(f"Acc@IoU=0.5 : {metrics['accuracy']:.4f} ({metrics['correct']}/{metrics['total']})") - print(f"Mean IoU : {metrics['mean_iou']:.4f}") - print(f"Unparsed : {metrics['unparsed']}") - if metrics.get("latency"): - lat = metrics["latency"] - print(f"Latency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " - f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms max={lat['max_ms']}ms") - print("\nIoU thresholds:") - for t, acc in metrics["iou_thresholds"].items(): - print(f" {t}: {acc:.4f}") - - -def build_tag(dataset: str, split: str) -> str: - ds_slug = dataset.split("/")[-1].lower().replace("+", "plus") - return f"kimi_k26_{ds_slug}_{split}" - - -async def run_predictions(dataset_name: str, split: str, pred_path: Path, limit: int | None): - print(f"Loading {dataset_name}, split={split}...") - dataset = load_dataset(dataset_name, split=split) - print(f"Loaded {len(dataset)} rows") - - samples = build_samples(dataset) - done_ids = load_completed_ids(pred_path) - pending = [s for s in samples if s["id"] not in done_ids] - if limit is not None: - pending = pending[:limit] - print(f"--limit applied: will run at most {limit} sample(s)") - print(f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})") - if not pending: - return - - asyncio.get_running_loop().set_default_executor( - concurrent.futures.ThreadPoolExecutor(max_workers=CONCURRENCY) - ) - - writer = JsonlWriter(pred_path) - semaphore = asyncio.Semaphore(CONCURRENCY) - progress = {"total": len(pending), "done": 0, "correct": 0, "failed": 0} - tasks = [process_sample(s, semaphore, writer, progress) for s in pending] - try: - await tqdm_asyncio.gather(*tasks, desc=f"{dataset_name}/{split}") - except Exception: - traceback.print_exc() - print(f"\nRun finished: {progress['done']}/{progress['total']} answered, " - f"{progress['correct']} correct, {progress['failed']} failed.") - - -def run_evaluation(dataset_name: str, split: str, pred_path: Path, metrics_path: Path): - if not pred_path.exists(): - print(f"No predictions found at {pred_path}") - sys.exit(1) - results = load_records(pred_path) - if not results: - print(f"No records in {pred_path}") - sys.exit(1) - - metrics = compute_metrics(results) - print_summary(metrics, dataset_name, split) - output = { - **metrics, - "dataset": dataset_name, "split": split, - "reasoning": "off", "temperature": TEMPERATURE, - "concurrency": CONCURRENCY, "iou_threshold": IOU_THRESHOLD, - "model": MODEL, "provider": "moonshotai (via openrouter)", - } - metrics_path.parent.mkdir(parents=True, exist_ok=True) - with open(metrics_path, "w") as f: - json.dump(output, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - parser = argparse.ArgumentParser( - description="RefCOCO benchmark for Kimi K2.6 via OpenRouter " - "(strict head-to-head with refcoco.py — same prompt, parser, IoU)" - ) - parser.add_argument("--dataset", default=DEFAULT_DATASET, - help="lmms-lab/RefCOCO | lmms-lab/RefCOCO+ | lmms-lab/RefCOCOg") - parser.add_argument("--split", default=DEFAULT_SPLIT, - help="val | testA | testB | test (availability varies by dataset)") - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument("--limit", type=int, default=None, - help="Only run the first N unanswered samples") - args = parser.parse_args() - - tag = build_tag(args.dataset, args.split) - pred_path = RESULTS_DIR / f"{tag}_responses.jsonl" - metrics_path = RESULTS_DIR / f"{tag}_metrics.json" - - if args.evaluate_only: - run_evaluation(args.dataset, args.split, pred_path, metrics_path) - elif args.predict_only: - asyncio.run(run_predictions(args.dataset, args.split, pred_path, limit=args.limit)) - else: - asyncio.run(run_predictions(args.dataset, args.split, pred_path, limit=args.limit)) - run_evaluation(args.dataset, args.split, pred_path, metrics_path) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/obj_detection/refcoco_multi.py b/benchmarks/obj_detection/refcoco_multi.py deleted file mode 100644 index f89f845..0000000 --- a/benchmarks/obj_detection/refcoco_multi.py +++ /dev/null @@ -1,598 +0,0 @@ -""" -RefCOCO benchmark against multiple VLM providers (OpenAI, Anthropic, Google). - -Designed for head-to-head comparison with the interfaze-beta numbers from -`refcoco.py` on the same splits (Acc@IoU=0.5). Each provider runs with -thinking/reasoning DISABLED to match the interfaze run's `reasoning_effort=None`. - -Usage: - # Single-model run: - uv run -m benchmarks.obj_detection.refcoco_multi --provider openai --model gpt-5.4 - uv run -m benchmarks.obj_detection.refcoco_multi --provider anthropic --model claude-sonnet-4-6 - uv run -m benchmarks.obj_detection.refcoco_multi --provider gemini --model gemini-3.0-flash - - # Smoke test with --limit 1: - uv run -m benchmarks.obj_detection.refcoco_multi --provider openai --model gpt-5.4 --limit 1 - -Keys are loaded from ~/interfaze/.env.local (OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_KEY). -""" - -import os -import sys -import json -import time -import base64 -import asyncio -import argparse -import traceback -from io import BytesIO -from pathlib import Path - -from datasets import load_dataset -from dotenv import load_dotenv -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -load_dotenv() - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -# Reuse parsing/IoU from the interfaze script — guarantees identical scoring. -from benchmarks.obj_detection.refcoco import ( # noqa: E402 - parse_box, - compute_iou, - coco_bbox_to_xyxy, - JsonlWriter, - build_samples, - load_completed_ids, - load_records, - compute_metrics, - print_summary, - PROMPT_TEMPLATE, - IOU_THRESHOLD, -) - - -class RateLimiter: - """Simple async token-bucket. Local to this script — refcoco.py does not - export one, so we don't try to import it.""" - - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -RESULTS_DIR = PROJECT_ROOT / "results" -DEFAULT_DATASET = "lmms-lab/RefCOCO" -DEFAULT_SPLIT = "testA" -RATE_LIMIT = 25 -MAX_RETRIES = 3 - -# Cap on requests in flight. RATE_LIMIT bounds how many requests *start* per -# second but not how many are outstanding: gather() launches every pending -# sample at once, so hundreds pile up, queue server-side, and eventually 429 -# after tens of minutes of latency. Override with BENCH_MAX_IN_FLIGHT. -MAX_IN_FLIGHT = int(os.getenv("BENCH_MAX_IN_FLIGHT", "8")) - -# reasoning.effort sent to OpenRouter-hosted models. Override per run. -OPENROUTER_EFFORT = os.getenv("BENCH_OPENROUTER_EFFORT", "none") - -def _gemini_floor(model: str) -> str: - """Lowest thinking_level the given Gemini model actually accepts. - - gemini-3.7+ flash rejects MINIMAL server-side ("Thinking level MINIMAL is - not supported for this model"), and the SDK enum has no "off"/"none", so - "low" is its true floor. Older 3.x flash models still take "minimal". - BENCH_GEMINI_FLOOR overrides for anything not covered here. - """ - override = os.getenv("BENCH_GEMINI_FLOOR") - if override: - return override - m = model.lower() - if any(v in m for v in ("3.7", "3.8", "3.9", "4.")): - return "low" - return "minimal" - - -def _load_interfaze_env() -> dict: - """Parse ~/interfaze/.env.local and return a dict of keys. Optional — the - project .env (loaded above) is enough for providers keyed there, so a - missing file is not fatal.""" - path = Path.home() / "interfaze" / ".env.local" - if not path.exists(): - return {} - env = {} - for line in path.read_text().splitlines(): - line = line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - k, v = line.split("=", 1) - v = v.strip().strip('"').strip("'") - env[k.strip()] = v - return env - - -# -------------------------------------------------------------------------- -# Provider adapters -# -------------------------------------------------------------------------- -# Each adapter is a function: (image_pil, prompt, model) -> (content, request_id) -# Runs synchronously inside asyncio.to_thread. - - -def _image_to_jpeg_bytes(image, max_side: int = 1024) -> tuple[bytes, int, int]: - if image.mode != "RGB": - image = image.convert("RGB") - w, h = image.size - scale = min(1.0, max_side / max(w, h)) - if scale < 1.0: - new_w, new_h = int(round(w * scale)), int(round(h * scale)) - image = image.resize((new_w, new_h)) - w, h = new_w, new_h - buf = BytesIO() - image.save(buf, format="JPEG", quality=92) - return buf.getvalue(), w, h - - -def call_openai(image, prompt: str, model: str, client) -> tuple[str, str, int, int]: - """Returns (content, request_id, width, height). - - For GPT-5.4 (and 5.2+) reasoning is OFF by default ("none"). We still pass - it explicitly to defend against future default changes. NOTE: "minimal" - is NOT a valid value on GPT-5.4 — valid are none/low/medium/high/xhigh. - """ - img_bytes, w, h = _image_to_jpeg_bytes(image) - b64 = base64.b64encode(img_bytes).decode("utf-8") - data_url = f"data:image/jpeg;base64,{b64}" - kwargs = { - "model": model, - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - {"type": "image_url", "image_url": {"url": data_url}}, - ], - } - ], - } - if model.startswith("gpt-5") or model.startswith("o"): - kwargs["reasoning_effort"] = "none" - resp = client.chat.completions.create(**kwargs) - content = (resp.choices[0].message.content or "").strip() - return content, resp.id, w, h - - -def call_anthropic(image, prompt: str, model: str, client) -> tuple[str, str, int, int]: - """Extended thinking explicitly disabled — belt-and-suspenders even though - omitting `thinking` is off by default for Sonnet 4.6.""" - img_bytes, w, h = _image_to_jpeg_bytes(image) - b64 = base64.b64encode(img_bytes).decode("utf-8") - resp = client.messages.create( - model=model, - max_tokens=1024, - thinking={"type": "disabled"}, - messages=[ - { - "role": "user", - "content": [ - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/jpeg", - "data": b64, - }, - }, - {"type": "text", "text": prompt}, - ], - } - ], - ) - parts = [b.text for b in resp.content if getattr(b, "type", None) == "text"] - content = "\n".join(parts).strip() - return content, resp.id, w, h - - -def call_openrouter(image, prompt: str, model: str, client) -> tuple[str, str, int, int]: - """OpenRouter-hosted VLMs. Reasoning is routed through extra_body, which is - OpenRouter's shape rather than the OpenAI `reasoning_effort` field. - - NOTE for thinkingmachines/*: on OpenRouter effort="none" genuinely disables - thinking (0 reasoning tokens), whereas the same value on Fireworks is only a - floor and still emits hundreds. The two hosts are not interchangeable at the - same nominal setting — see README. - """ - img_bytes, w, h = _image_to_jpeg_bytes(image) - b64 = base64.b64encode(img_bytes).decode("utf-8") - resp = client.chat.completions.create( - model=model, - messages=[ - { - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - { - "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64}"}, - }, - ], - } - ], - temperature=0.0, - extra_body={"reasoning": {"effort": OPENROUTER_EFFORT}}, - ) - content = (resp.choices[0].message.content or "").strip() - return content, getattr(resp, "id", "") or "", w, h - - -def call_fireworks(image, prompt: str, model: str, client) -> tuple[str, str, int, int]: - """Fireworks-hosted models (Inkling). Inkling always thinks; "none" is the - reasoning floor, which is the closest match to the interfaze run's - reasoning_effort=None.""" - img_bytes, w, h = _image_to_jpeg_bytes(image) - b64 = base64.b64encode(img_bytes).decode("utf-8") - resp = client.chat.completions.create( - model=model, - messages=[ - { - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - { - "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64}"}, - }, - ], - } - ], - temperature=0.0, - reasoning_effort="none", - ) - content = (resp.choices[0].message.content or "").strip() - return content, getattr(resp, "id", "") or "", w, h - - -def call_gemini(image, prompt: str, model: str, client) -> tuple[str, str, int, int]: - """Gemini 3.x has thinking ON by default (HIGH for Pro, can't disable). - We always override to the LOWEST supported level for the chosen model: - gemini-3*-pro*: 'low' (Pro rejects 'minimal'; min is 'low') - gemini-3*-flash*: 'minimal' (Flash supports 'minimal' as the floor) - Gemini 2.5 used thinking_budget; we don't support that path here.""" - from google.genai import types - - img_bytes, w, h = _image_to_jpeg_bytes(image) - m = model.lower() - if "pro" in m: - thinking_level = "low" - else: - thinking_level = _gemini_floor(model) - config = types.GenerateContentConfig( - temperature=0.0, - thinking_config=types.ThinkingConfig(thinking_level=thinking_level), - ) - resp = client.models.generate_content( - model=model, - contents=[ - types.Part.from_bytes(data=img_bytes, mime_type="image/jpeg"), - prompt, - ], - config=config, - ) - content = (resp.text or "").strip() - request_id = getattr(resp, "response_id", None) or "" - return content, request_id, w, h - - -# -------------------------------------------------------------------------- -# Provider-agnostic pipeline (mirrors refcoco.py's process_sample) -# -------------------------------------------------------------------------- - - -async def process_sample( - sample: dict, - call_fn, - model: str, - rate_limiter, - writer: JsonlWriter, - progress: dict, - provider: str, - client, -) -> dict | None: - """Pre-resize the image to know the exact dims that will be sent, embed - those dims in the prompt, then call the provider adapter.""" - tmp_bytes, sent_w, sent_h = _image_to_jpeg_bytes(sample["image"]) - del tmp_bytes - orig_w, orig_h = sample["image"].size - sx = sent_w / orig_w - sy = sent_h / orig_h - gt_xyxy = coco_bbox_to_xyxy(sample["bbox_xywh"], sx, sy) - - prompt = PROMPT_TEMPLATE.format( - width=sent_w, height=sent_h, expression=sample["expression"] - ) - - last_error: str | None = None - for attempt in range(1, MAX_RETRIES + 1): - await rate_limiter.acquire() - start = time.perf_counter() - try: - content, request_id, _, _ = await asyncio.to_thread( - call_fn, sample["image"], prompt, model, client - ) - latency_ms = int((time.perf_counter() - start) * 1000) - if not content: - last_error = "empty response content" - raise RuntimeError(last_error) - - pred_box = parse_box(content, sent_w, sent_h) - iou = compute_iou(pred_box, gt_xyxy) if pred_box else 0.0 - correct = pred_box is not None and iou >= IOU_THRESHOLD - - record = { - "id": sample["id"], - "question_id": sample["question_id"], - "file_name": sample["file_name"], - "expression": sample["expression"], - "all_expressions": sample["all_expressions"], - "image_width": sent_w, - "image_height": sent_h, - "gt_bbox_xyxy": gt_xyxy, - "pred_bbox_xyxy": pred_box, - "iou": iou, - "correct": correct, - "response": content, - "request_id": request_id, - "latency_ms": latency_ms, - "attempts": attempt, - "provider": provider, - "model": model, - } - await writer.append(record) - progress["done"] += 1 - if correct: - progress["correct"] += 1 - mark = "OK" - else: - mark = "X " - tqdm.write( - f"[{provider} {progress['done']}/{progress['total']}] {mark} " - f"id={sample['id']} iou={iou:.3f} " - f"pred={pred_box} gt={[round(x, 1) for x in gt_xyxy]} " - f"latency={latency_ms}ms req_id={request_id} attempt={attempt}" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - tqdm.write( - f"[{provider} error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(2 ** (attempt - 1)) - - progress["failed"] += 1 - tqdm.write( - f"[{provider} FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}" - ) - return None - - -# -------------------------------------------------------------------------- -# CLI -# -------------------------------------------------------------------------- - - -def build_tag(dataset: str, split: str, provider: str, model: str) -> str: - ds_slug = dataset.split("/")[-1].lower().replace("+", "plus") - model_slug = model.replace("/", "_").replace(":", "_") - return f"{ds_slug}_{split}_{provider}_{model_slug}" - - -def build_client(provider: str, env: dict): - if provider == "openai": - from openai import OpenAI - - return OpenAI(api_key=env["OPENAI_API_KEY"]) - if provider == "anthropic": - from anthropic import Anthropic - - return Anthropic(api_key=env["ANTHROPIC_API_KEY"]) - if provider == "gemini": - from google import genai - - api_key = ( - env.get("GEMINI_KEY") - or os.getenv("GEMINI_API_KEY") - or os.getenv("GEMINI_KEY") - or os.getenv("GOOGLE_API_KEY") - ) - if not api_key: - raise RuntimeError("GEMINI_API_KEY missing from .env") - return genai.Client(api_key=api_key) - if provider == "openrouter": - from openai import OpenAI - - api_key = ( - env.get("OPENROUTER_API_KEY") - or os.getenv("OPENROUTER_API_KEY") - or os.getenv("openrouter_api_key") - ) - if not api_key: - raise RuntimeError("OPENROUTER_API_KEY missing from .env") - return OpenAI(base_url="https://openrouter.ai/api/v1", api_key=api_key) - if provider == "fireworks": - from openai import OpenAI - - # .env in this repo may spell the key lowercase; accept either casing. - api_key = ( - env.get("FIREWORKS_API_KEY") - or os.getenv("FIREWORKS_API_KEY") - or os.getenv("fireworks_api_key") - ) - if not api_key: - raise RuntimeError("FIREWORKS_API_KEY missing from .env") - return OpenAI(base_url="https://api.fireworks.ai/inference/v1", api_key=api_key) - raise ValueError(f"unknown provider: {provider}") - - -def get_call_fn(provider: str): - return { - "openai": call_openai, - "anthropic": call_anthropic, - "gemini": call_gemini, - "fireworks": call_fireworks, - "openrouter": call_openrouter, - }[provider] - - -async def run( - provider: str, - model: str, - dataset_name: str, - split: str, - pred_path: Path, - limit: int | None, -): - env = _load_interfaze_env() - client = build_client(provider, env) - call_fn = get_call_fn(provider) - - print(f"[{provider}/{model}] Loading {dataset_name}, split={split}...") - dataset = load_dataset(dataset_name, split=split) - samples = build_samples(dataset) - - done_ids = load_completed_ids(pred_path) - pending = [s for s in samples if s["id"] not in done_ids] - if limit is not None: - pending = pending[:limit] - print(f"--limit applied: will run at most {limit} sample(s)") - print( - f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})" - ) - if not pending: - return - - writer = JsonlWriter(pred_path) - rate_limiter = RateLimiter(RATE_LIMIT) - progress = {"total": len(pending), "done": 0, "correct": 0, "failed": 0} - sem = asyncio.Semaphore(MAX_IN_FLIGHT) - - async def bounded(sample): - async with sem: - return await process_sample( - sample, call_fn, model, rate_limiter, writer, progress, provider, client - ) - - tasks = [bounded(s) for s in pending] - try: - await tqdm_asyncio.gather(*tasks, desc=f"{provider}/{model}") - except Exception: - traceback.print_exc() - print( - f"\n[{provider}/{model}] Run finished: {progress['done']}/{progress['total']} answered, " - f"{progress['correct']} correct, {progress['failed']} failed." - ) - - -def run_evaluation( - dataset_name: str, - split: str, - pred_path: Path, - metrics_path: Path, - provider: str, - model: str, -): - if not pred_path.exists(): - print(f"No predictions found at {pred_path}") - sys.exit(1) - results = load_records(pred_path) - if not results: - print(f"No records in {pred_path}") - sys.exit(1) - metrics = compute_metrics(results) - print_summary(metrics, dataset_name, split) - output = { - **metrics, - "dataset": dataset_name, - "split": split, - "provider": provider, - "model": model, - "rate_limit": RATE_LIMIT, - "iou_threshold": IOU_THRESHOLD, - } - metrics_path.parent.mkdir(parents=True, exist_ok=True) - with open(metrics_path, "w") as f: - json.dump(output, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - parser = argparse.ArgumentParser(description="Multi-provider RefCOCO eval") - parser.add_argument( - "--provider", - required=True, - choices=["openai", "anthropic", "gemini", "fireworks", "openrouter"], - ) - parser.add_argument("--model", required=True, help="Provider-specific model id") - parser.add_argument("--dataset", default=DEFAULT_DATASET) - parser.add_argument("--split", default=DEFAULT_SPLIT) - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument("--limit", type=int, default=None) - args = parser.parse_args() - - tag = build_tag(args.dataset, args.split, args.provider, args.model) - pred_path = RESULTS_DIR / f"{tag}_responses.jsonl" - metrics_path = RESULTS_DIR / f"{tag}_metrics.json" - - if args.evaluate_only: - run_evaluation( - args.dataset, args.split, pred_path, metrics_path, args.provider, args.model - ) - elif args.predict_only: - asyncio.run( - run( - args.provider, - args.model, - args.dataset, - args.split, - pred_path, - limit=args.limit, - ) - ) - else: - asyncio.run( - run( - args.provider, - args.model, - args.dataset, - args.split, - pred_path, - limit=args.limit, - ) - ) - run_evaluation( - args.dataset, args.split, pred_path, metrics_path, args.provider, args.model - ) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/obj_detection/refcoco_openai.py b/benchmarks/obj_detection/refcoco_openai.py deleted file mode 100644 index 3f9d071..0000000 --- a/benchmarks/obj_detection/refcoco_openai.py +++ /dev/null @@ -1,306 +0,0 @@ -""" -RefCOCO benchmark for OpenAI GPT-5 series. - -Strict head-to-head with `refcoco.py` (interfaze): same prompt, same parser, -same IoU, same checkpointing — only the inference call changes. - -Default model: `gpt-5.5`. Other models can be selected with `--model`, e.g. -`--model gpt-5.4-mini`. Reasoning defaults to fully off (`reasoning_effort="none"`), -which is accepted by the GPT-5.x family. Temperature pinned to 0.0. - -Usage: - uv run -m benchmarks.obj_detection.refcoco_openai --split testA - uv run -m benchmarks.obj_detection.refcoco_openai --split testA --model gpt-5.4-mini - uv run -m benchmarks.obj_detection.refcoco_openai --split testA --limit 20 - uv run -m benchmarks.obj_detection.refcoco_openai --split testA --evaluate-only - -Env: OPENAI_API_KEY must be set (loaded from .env). -""" - -import re -import sys -import json -import os -import time -import asyncio -import argparse -import traceback -import concurrent.futures -from pathlib import Path - -from datasets import load_dataset -from dotenv import load_dotenv -from openai import OpenAI -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -from benchmarks.obj_detection.refcoco import ( # noqa: E402 - JsonlWriter, - PROMPT_TEMPLATE, - build_samples, - coco_bbox_to_xyxy, - compute_iou, - compute_metrics, - load_completed_ids, - load_records, - parse_box, - pil_to_data_url, -) - -load_dotenv() - -RESULTS_DIR = PROJECT_ROOT / "results" -DEFAULT_DATASET = "lmms-lab/RefCOCO" -DEFAULT_SPLIT = "val" -DEFAULT_MODEL = "gpt-5.5" -DEFAULT_REASONING_EFFORT = "none" -TEMPERATURE = 0.0 -CONCURRENCY = 10 -MAX_RETRIES = 5 -RETRY_BACKOFF_CAP_S = 30.0 -IOU_THRESHOLD = 0.5 - -# Mutated by CLI before any inference runs. -MODEL = DEFAULT_MODEL -REASONING_EFFORT = DEFAULT_REASONING_EFFORT - - -def model_slug(model: str) -> str: - """Compact model id used in output filenames. Matches existing convention: - 'gpt-5.5' -> 'gpt55', 'gpt-5.4-mini' -> 'gpt54mini'.""" - return re.sub(r"[^a-z0-9]", "", model.lower()) - -OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") -if not OPENAI_API_KEY: - raise RuntimeError( - "OPENAI_API_KEY is not set. Add it to .env " - "(get one from https://platform.openai.com/api-keys)." - ) - -# Override OPENAI_BASE_URL — `.env` points it at the interfaze worker for the -# main benchmark; we need to hit the real OpenAI API here. -openai_client = OpenAI( - base_url="https://api.openai.com/v1", - api_key=OPENAI_API_KEY, -) - - -def invoke_openai(messages: list[dict]): - """Single chat.completions call against the selected GPT-5.x model. - - GPT-5.x rejects non-default temperature when reasoning is engaged. We pass - `temperature=0.0` only when reasoning is fully off ('none'); otherwise we - omit it and let the API default apply.""" - kwargs = { - "model": MODEL, - "messages": messages, - "reasoning_effort": REASONING_EFFORT, - } - if REASONING_EFFORT == "none": - kwargs["temperature"] = TEMPERATURE - return openai_client.chat.completions.create(**kwargs) - - -async def process_sample(sample: dict, semaphore: asyncio.Semaphore, - writer: JsonlWriter, progress: dict) -> dict | None: - orig_w, orig_h = sample["image"].size - data_url, sent_w, sent_h = pil_to_data_url(sample["image"]) - sx = sent_w / orig_w - sy = sent_h / orig_h - gt_xyxy = coco_bbox_to_xyxy(sample["bbox_xywh"], sx, sy) - - prompt = PROMPT_TEMPLATE.format( - width=sent_w, height=sent_h, expression=sample["expression"] - ) - messages = [{ - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - {"type": "image_url", "image_url": {"url": data_url}}, - ], - }] - last_error: str | None = None - - for attempt in range(1, MAX_RETRIES + 1): - start = time.perf_counter() - try: - async with semaphore: - start = time.perf_counter() - response = await asyncio.to_thread(invoke_openai, messages) - latency_ms = int((time.perf_counter() - start) * 1000) - content = (response.choices[0].message.content or "").strip() - request_id = getattr(response, "id", None) - if not content: - last_error = "empty response content" - raise RuntimeError(last_error) - - pred_box = parse_box(content, sent_w, sent_h) - iou = compute_iou(pred_box, gt_xyxy) if pred_box else 0.0 - correct = pred_box is not None and iou >= IOU_THRESHOLD - - record = { - "id": sample["id"], - "question_id": sample["question_id"], - "file_name": sample["file_name"], - "expression": sample["expression"], - "all_expressions": sample["all_expressions"], - "image_width": sent_w, - "image_height": sent_h, - "gt_bbox_xyxy": gt_xyxy, - "pred_bbox_xyxy": pred_box, - "iou": iou, - "correct": correct, - "response": content, - "request_id": request_id, - "latency_ms": latency_ms, - "attempts": attempt, - } - await writer.append(record) - - progress["done"] += 1 - if correct: - progress["correct"] += 1 - mark = "OK" - else: - mark = "X " - tqdm.write( - f"[{progress['done']}/{progress['total']}] {mark} " - f"id={sample['id']} iou={iou:.3f} " - f"pred={pred_box} gt={[round(x,1) for x in gt_xyxy]} " - f"latency={latency_ms}ms req_id={request_id} attempt={attempt}" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - tqdm.write( - f"[error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(min(2 ** (attempt - 1), RETRY_BACKOFF_CAP_S)) - - progress["failed"] += 1 - tqdm.write(f"[FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") - return None - - -def print_summary(metrics: dict, dataset_name: str, split: str): - print(f"\n{'=' * 60}") - print(f"Grounding Results — {dataset_name} / {split} ({MODEL}, reasoning={REASONING_EFFORT})") - print(f"{'=' * 60}") - print(f"Acc@IoU=0.5 : {metrics['accuracy']:.4f} ({metrics['correct']}/{metrics['total']})") - print(f"Mean IoU : {metrics['mean_iou']:.4f}") - print(f"Unparsed : {metrics['unparsed']}") - if metrics.get("latency"): - lat = metrics["latency"] - print(f"Latency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " - f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms max={lat['max_ms']}ms") - print("\nIoU thresholds:") - for t, acc in metrics["iou_thresholds"].items(): - print(f" {t}: {acc:.4f}") - - -def build_tag(dataset: str, split: str) -> str: - ds_slug = dataset.split("/")[-1].lower().replace("+", "plus") - return f"{model_slug(MODEL)}_{ds_slug}_{split}" - - -async def run_predictions(dataset_name: str, split: str, pred_path: Path, limit: int | None): - print(f"Loading {dataset_name}, split={split}...") - dataset = load_dataset(dataset_name, split=split) - print(f"Loaded {len(dataset)} rows") - - samples = build_samples(dataset) - done_ids = load_completed_ids(pred_path) - pending = [s for s in samples if s["id"] not in done_ids] - if limit is not None: - pending = pending[:limit] - print(f"--limit applied: will run at most {limit} sample(s)") - print(f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})") - if not pending: - return - - asyncio.get_running_loop().set_default_executor( - concurrent.futures.ThreadPoolExecutor(max_workers=CONCURRENCY) - ) - - writer = JsonlWriter(pred_path) - semaphore = asyncio.Semaphore(CONCURRENCY) - progress = {"total": len(pending), "done": 0, "correct": 0, "failed": 0} - tasks = [process_sample(s, semaphore, writer, progress) for s in pending] - try: - await tqdm_asyncio.gather(*tasks, desc=f"{dataset_name}/{split}") - except Exception: - traceback.print_exc() - print(f"\nRun finished: {progress['done']}/{progress['total']} answered, " - f"{progress['correct']} correct, {progress['failed']} failed.") - - -def run_evaluation(dataset_name: str, split: str, pred_path: Path, metrics_path: Path): - if not pred_path.exists(): - print(f"No predictions found at {pred_path}") - sys.exit(1) - results = load_records(pred_path) - if not results: - print(f"No records in {pred_path}") - sys.exit(1) - - metrics = compute_metrics(results) - print_summary(metrics, dataset_name, split) - output = { - **metrics, - "dataset": dataset_name, "split": split, - "reasoning_effort": REASONING_EFFORT, "temperature": TEMPERATURE, - "concurrency": CONCURRENCY, "iou_threshold": IOU_THRESHOLD, - "model": MODEL, - } - metrics_path.parent.mkdir(parents=True, exist_ok=True) - with open(metrics_path, "w") as f: - json.dump(output, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - global MODEL, REASONING_EFFORT - parser = argparse.ArgumentParser( - description="RefCOCO benchmark for OpenAI GPT-5.x " - "(strict head-to-head with refcoco.py — same prompt, parser, IoU)" - ) - parser.add_argument("--dataset", default=DEFAULT_DATASET, - help="lmms-lab/RefCOCO | lmms-lab/RefCOCO+ | lmms-lab/RefCOCOg") - parser.add_argument("--split", default=DEFAULT_SPLIT, - help="val | testA | testB | test (availability varies by dataset)") - parser.add_argument("--model", default=DEFAULT_MODEL, - help="OpenAI model id (e.g. gpt-5.5, gpt-5.4-mini)") - parser.add_argument("--reasoning-effort", default=DEFAULT_REASONING_EFFORT, - help="reasoning_effort param ('none', 'minimal', 'low', 'medium', 'high', etc.)") - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument("--limit", type=int, default=None, - help="Only run the first N unanswered samples") - args = parser.parse_args() - - MODEL = args.model - REASONING_EFFORT = args.reasoning_effort - - tag = build_tag(args.dataset, args.split) - pred_path = RESULTS_DIR / f"{tag}_responses.jsonl" - metrics_path = RESULTS_DIR / f"{tag}_metrics.json" - - if args.evaluate_only: - run_evaluation(args.dataset, args.split, pred_path, metrics_path) - elif args.predict_only: - asyncio.run(run_predictions(args.dataset, args.split, pred_path, limit=args.limit)) - else: - asyncio.run(run_predictions(args.dataset, args.split, pred_path, limit=args.limit)) - run_evaluation(args.dataset, args.split, pred_path, metrics_path) - - -if __name__ == "__main__": - main() diff --git a/scripts/run_fireworks_suite.sh b/scripts/run_fireworks_suite.sh index 3cc37a5..6dabcf3 100755 --- a/scripts/run_fireworks_suite.sh +++ b/scripts/run_fireworks_suite.sh @@ -86,8 +86,12 @@ else echo " [skip] mmmupro: set TARGET= (python -m bench_core list-targets)" fi -run_step refcoco uv run -m benchmarks.obj_detection.refcoco_multi \ - --provider "$PROVIDER" --model "$MODEL" --split "$REFCOCO_SPLIT" +if [ -n "${TARGET:-}" ]; then + run_step refcoco uv run python -m bench_core run --benchmark refcoco \ + --target "$TARGET" --variant "$REFCOCO_SPLIT" +else + echo " [skip] refcoco: set TARGET= (python -m bench_core list-targets)" +fi run_step olmocr uv run -m benchmarks.olmocr.olmocr_bench_fireworks --model "$MODEL" @@ -111,8 +115,7 @@ else echo " [skip] gpqa: set TARGET= (python -m bench_core list-targets)" fi -# Grounding is scored strictly above; this adds the repo's uniform -# format-tolerant oracle pass (upper bound) for every RefCOCO run present. -run_step refcoco_oracle uv run -m benchmarks.obj_detection.reeval_format_tolerant +# The format-tolerant oracle (upper bound) is now computed inline by the refcoco +# run and stored under submetrics.oracle in its metrics.json — no separate step. echo "=== [$(date +%H:%M:%S)] suite finished for $MODEL" diff --git a/tests/test_refcoco_bench.py b/tests/test_refcoco_bench.py new file mode 100644 index 0000000..b5ab977 --- /dev/null +++ b/tests/test_refcoco_bench.py @@ -0,0 +1,89 @@ +"""RefCOCO: parse/IoU/oracle units + offline scoring-parity replay. + +Strict replay re-parses each archived response and matches Acc@0.5; the oracle +replay re-runs best-of-interpretation and matches the archived _oracle_ metrics. +""" + +import json +import pathlib + +import pytest + +from benchmarks.obj_detection import bench + +RESULTS = pathlib.Path(__file__).resolve().parent.parent / "results" +STRICT_TAGS = [ + "refcoco_val_fireworks_accounts_fireworks_models_inkling", + "refcoco_val_openrouter_thinkingmachines_inkling-small", + "refcoco_testA_gemini_gemini-3.7-flash", +] +ORACLE_BASE = "refcoco_val_fireworks_accounts_fireworks_models_inkling" + + +def test_compute_iou(): + assert bench.compute_iou([0, 0, 10, 10], [0, 0, 10, 10]) == 1.0 + assert bench.compute_iou([0, 0, 10, 10], [20, 20, 30, 30]) == 0.0 + + +def test_parse_box_pixel_and_box2d(): + assert bench.parse_box("[10, 20, 30, 40]", 100, 100) == [10, 20, 30, 40] + # box_2d is yxyx -> swapped to xyxy + assert bench.parse_box('{"box_2d": [20, 10, 40, 30]}', 1000, 1000) == pytest.approx( + [10, 20, 30, 40] + ) + + +def test_best_box_recovers_pixel2x(): + # a box reported in 2x-upscaled space; oracle should halve it to match GT + iou, label = bench._best_box("[100, 100, 200, 200]", 100, 100, [50, 50, 100, 100]) + assert label == "pixel2x-xyxy" + assert iou == 1.0 + + +def _replay(tag): + resp_path = RESULTS / f"{tag}_responses.jsonl" + samples, records = [], [] + for line in resp_path.read_text().splitlines(): + if not line.strip(): + continue + rec = json.loads(line) + w, h = rec["image_width"], rec["image_height"] + samples.append( + { + "id": rec["id"], + "gt_bbox_xyxy": rec["gt_bbox_xyxy"], + "sent_w": w, + "sent_h": h, + } + ) + records.append( + { + "id": rec["id"], + "response": rec["response"], + "prediction": bench.parse_box(rec["response"], w, h), + } + ) + return bench.score(records, samples) + + +@pytest.mark.parametrize("tag", STRICT_TAGS) +def test_strict_scoring_parity(tag): + resp_path = RESULTS / f"{tag}_responses.jsonl" + metrics_path = RESULTS / f"{tag}_metrics.json" + if not (resp_path.exists() and metrics_path.exists()): + pytest.skip(f"archived run {tag} not present") + archived = json.loads(metrics_path.read_text()) + m = _replay(tag) + assert m["total"] == archived["total"] + assert m["accuracy"] == pytest.approx(archived["accuracy"]) + assert m["unparsed"] == archived["unparsed"] + + +def test_oracle_scoring_parity(): + resp_path = RESULTS / f"{ORACLE_BASE}_responses.jsonl" + oracle_metrics = RESULTS / f"{ORACLE_BASE}_oracle_metrics.json" + if not (resp_path.exists() and oracle_metrics.exists()): + pytest.skip("archived oracle run not present") + archived = json.loads(oracle_metrics.read_text()) + m = _replay(ORACLE_BASE) + assert m["oracle"]["accuracy"] == pytest.approx(archived["accuracy"]) From f0c4be94423248349b6cfbf9ed18be010a750f4e Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 08:53:38 +0530 Subject: [PATCH 12/38] feat(bench_core): migrate OCRBench v2 onto the abstraction; retire 10 forks benchmarks/ocrbench_v2/bench.py routes inference through bench_core and reuses the existing eval_scripts scorer verbatim (file-based, chdir, RRC shell); only the macro-of-category aggregation is ported. Retires the 10 per-provider/per-task runners (ocrbench_v2.py + 9 variants); eval_scripts/ kept as the scorer. Offline parity replays the archived scored.json through the ported aggregate and matches en_overall/cn_overall + every category (inkling 0.5785, inkling-small 0.5659, gemini 0.7134). Images decode lazily per request (no 10k-image OOM). 150 tests pass. --- README.md | 21 +- bench_core/cli.py | 1 + benchmarks/ocrbench_v2/bench.py | 207 ++++++++ benchmarks/ocrbench_v2/ocrbench_v2.py | 350 -------------- .../ocrbench_v2/ocrbench_v2_anthropic.py | 355 -------------- .../ocrbench_v2/ocrbench_v2_fireworks.py | 441 ------------------ benchmarks/ocrbench_v2/ocrbench_v2_gemini.py | 346 -------------- .../ocrbench_v2/ocrbench_v2_gemini_pro_31.py | 342 -------------- benchmarks/ocrbench_v2/ocrbench_v2_grok.py | 347 -------------- benchmarks/ocrbench_v2/ocrbench_v2_kimi.py | 346 -------------- benchmarks/ocrbench_v2/ocrbench_v2_openai.py | 342 -------------- .../ocrbench_v2/ocrbench_v2_openai_mini.py | 342 -------------- .../ocrbench_v2_text_spotting_en.py | 376 --------------- scripts/run_fireworks_suite.sh | 7 +- tests/test_ocrbench_bench.py | 53 +++ 15 files changed, 271 insertions(+), 3605 deletions(-) create mode 100644 benchmarks/ocrbench_v2/bench.py delete mode 100644 benchmarks/ocrbench_v2/ocrbench_v2.py delete mode 100644 benchmarks/ocrbench_v2/ocrbench_v2_anthropic.py delete mode 100644 benchmarks/ocrbench_v2/ocrbench_v2_fireworks.py delete mode 100644 benchmarks/ocrbench_v2/ocrbench_v2_gemini.py delete mode 100644 benchmarks/ocrbench_v2/ocrbench_v2_gemini_pro_31.py delete mode 100644 benchmarks/ocrbench_v2/ocrbench_v2_grok.py delete mode 100644 benchmarks/ocrbench_v2/ocrbench_v2_kimi.py delete mode 100644 benchmarks/ocrbench_v2/ocrbench_v2_openai.py delete mode 100644 benchmarks/ocrbench_v2/ocrbench_v2_openai_mini.py delete mode 100644 benchmarks/ocrbench_v2/ocrbench_v2_text_spotting_en.py create mode 100644 tests/test_ocrbench_bench.py diff --git a/README.md b/README.md index 9535dcc..9ab86cb 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ models over their OpenAI-compatible endpoint. Inkling takes text, image, and audio input, so it runs on every benchmark here: ```bash -uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks +uv run python -m bench_core run --target inkling --benchmark ocrbench_v2 uv run -m benchmarks.olmocr.olmocr_bench_fireworks uv run python -m bench_core run --target inkling --benchmark gpqa uv run python -m bench_core run --target inkling --benchmark mmmlu @@ -139,21 +139,10 @@ Two caveats when reading Fireworks numbers: Links: [paper](https://arxiv.org/abs/2501.00321) · [repo](https://github.com/Yuliang-Liu/MultimodalOCR/tree/main/OCRBench_v2) ```bash -# Interfaze -uv run -m benchmarks.ocrbench_v2.ocrbench_v2 - -# Per-provider runners -uv run -m benchmarks.ocrbench_v2.ocrbench_v2_openai -uv run -m benchmarks.ocrbench_v2.ocrbench_v2_openai_mini -uv run -m benchmarks.ocrbench_v2.ocrbench_v2_anthropic -uv run -m benchmarks.ocrbench_v2.ocrbench_v2_gemini -uv run -m benchmarks.ocrbench_v2.ocrbench_v2_gemini_pro_31 -uv run -m benchmarks.ocrbench_v2.ocrbench_v2_grok -uv run -m benchmarks.ocrbench_v2.ocrbench_v2_kimi # via OpenRouter -uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks # Inkling - -# Text-spotting EN subset only -uv run -m benchmarks.ocrbench_v2.ocrbench_v2_text_spotting_en +# Any target (10k tasks, macro-of-category en_overall) +uv run python -m bench_core run --target gpt-5.5 --benchmark ocrbench_v2 +uv run python -m bench_core run --target gemini-3.7-flash --benchmark ocrbench_v2 +uv run python -m bench_core run --target inkling --benchmark ocrbench_v2 # Evaluate without re-running predictions uv run -m benchmarks.ocrbench_v2.ocrbench_v2 --evaluate-only diff --git a/bench_core/cli.py b/bench_core/cli.py index 24cd429..b2574e9 100644 --- a/bench_core/cli.py +++ b/bench_core/cli.py @@ -22,6 +22,7 @@ "mmmlu": "benchmarks.mmmlu.bench", "mmmu_pro": "benchmarks.mmmu_pro.bench", "refcoco": "benchmarks.obj_detection.bench", + "ocrbench_v2": "benchmarks.ocrbench_v2.bench", } diff --git a/benchmarks/ocrbench_v2/bench.py b/benchmarks/ocrbench_v2/bench.py new file mode 100644 index 0000000..a0a49c8 --- /dev/null +++ b/benchmarks/ocrbench_v2/bench.py @@ -0,0 +1,207 @@ +"""OCRBench v2: 10k OCR/vision tasks across 30 types; macro-of-category-means. + +The per-sample scorer (eval_scripts/eval.py) is reused verbatim — it is file- +based, needs CWD=benchmarks/ocrbench_v2 and eval_scripts on sys.path, and shells +out for text-spotting. Only the category aggregation is ported here. +""" + +from __future__ import annotations + +import json +import os +import sys +import tempfile +from collections import defaultdict +from pathlib import Path + +from bench_core.media import encode_image +from bench_core.request import Message, ReasoningSpec, Request, TextPart + +NAME = "ocrbench_v2" +ID_KEY = "id" +PRIMARY_METRIC = "en_overall" +DEFAULTS = {"reasoning": "off", "rate_limit": 25, "max_in_flight": 8} + +_BENCH_DIR = Path(__file__).resolve().parent +_DATASET_ID = "lmms-lab/OCRBench-v2" +_SPLIT = "test" + +TEXT_SPOTTING_PROMPT_TEMPLATE = """Use OCR on this image to spot all text at {level}. The OCR tool returns each detected text region with its text content and four corner coordinates: top_left, top_right, bottom_left, bottom_right (each as an x,y pixel pair). + +Then use run code to write a Python script that takes those OCR results and: +1. For each text region, compute the axis-aligned bounding box from the four corners: + - x1 = min of all x coordinates (leftmost) + - y1 = min of all y coordinates (topmost) + - x2 = max of all x coordinates (rightmost) + - y2 = max of all y coordinates (bottommost) +2. Normalize each coordinate to the range 0-1000 by dividing by the image width (for x) or height (for y) and multiplying by 1000, then rounding to an integer. +3. Print the results as a Python list. + +Your final answer must be ONLY a Python list in this exact format, with no markdown, no code fences, no explanation: +[(x1, y1, x2, y2, "text"), (x1, y1, x2, y2, "text"), ...]""" + +TYPE_TO_EN = { + "text recognition en": "text_recognition", + "fine-grained text recognition en": "text_recognition", + "full-page OCR en": "text_recognition", + "text grounding en": "text_detection", + "VQA with position en": "text_detection", + "text spotting en": "text_spotting", + "key information extraction en": "relationship_extraction", + "key information mapping en": "relationship_extraction", + "document parsing en": "element_parsing", + "chart parsing en": "element_parsing", + "table parsing en": "element_parsing", + "formula recognition en": "element_parsing", + "math QA en": "mathematical_calculation", + "text counting en": "mathematical_calculation", + "document classification en": "visual_text_understanding", + "cognition VQA en": "visual_text_understanding", + "diagram QA en": "visual_text_understanding", + "reasoning VQA en": "knowledge_reasoning", + "science QA en": "knowledge_reasoning", + "APP agent en": "knowledge_reasoning", + "ASCII art classification en": "knowledge_reasoning", +} +TYPE_TO_CN = { + "full-page OCR cn": "text_recognition", + "key information extraction cn": "relationship_extraction", + "handwritten answer extraction cn": "relationship_extraction", + "document parsing cn": "element_parsing", + "table parsing cn": "element_parsing", + "formula recognition cn": "element_parsing", + "cognition VQA cn": "visual_text_understanding", + "reasoning VQA cn": "knowledge_reasoning", + "text translation cn": "knowledge_reasoning", +} +# distinct categories in first-seen order +EN_CATEGORIES = list(dict.fromkeys(TYPE_TO_EN.values())) +CN_CATEGORIES = list(dict.fromkeys(TYPE_TO_CN.values())) + +_DATASET = None # lazily-held so images decode per request, not all 10k upfront + + +def get_spotting_prompt(question: str) -> str: + level = "line-level" if "line-level" in question else "word-level" + return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level=level) + + +def load_samples(sample_size: int | None = None) -> list[dict]: + global _DATASET + from datasets import load_dataset + + ds = load_dataset(_DATASET_ID, split=_SPLIT) + _DATASET = ds + meta = ds.select_columns(["id", "dataset_name", "type", "question", "answers"]) + n = len(meta) if sample_size is None else min(sample_size, len(meta)) + samples = [] + for i in range(n): + row = meta[i] + question = row["question"] + if row["type"] == "text spotting en": + question = get_spotting_prompt(question) + samples.append( + { + "id": row["id"], + "idx": i, + "dataset_name": row["dataset_name"], + "type": row["type"], + "question": question, + "answers": row["answers"], + } + ) + return samples + + +def build_request(sample: dict, mode: str) -> Request: + image = _DATASET[sample["idx"]]["image"] # decoded lazily + img = encode_image( + image, "image/jpeg" + ) # RGB + JPEG q95, no resize (matches runner) + return Request( + [Message("user", [TextPart(sample["question"]), img])], + reasoning=ReasoningSpec(mode), + temperature=0.0, + ) + + +def parse(response, sample) -> str: + return response.text or "" + + +def _run_per_sample_scorer(preds: list[dict]) -> list[dict]: + """Reuse eval_scripts/eval.py verbatim: it takes file paths, needs + CWD=benchmarks/ocrbench_v2 and eval_scripts on sys.path (text-spotting shells + out to relative dirs there).""" + eval_dir = _BENCH_DIR / "eval_scripts" + if str(eval_dir) not in sys.path: + sys.path.insert(0, str(eval_dir)) + from benchmarks.ocrbench_v2.eval_scripts.eval import process_predictions + + with tempfile.TemporaryDirectory() as td: + pred_path = Path(td) / "pred.json" + scored_path = Path(td) / "scored.json" + pred_path.write_text(json.dumps(preds, ensure_ascii=False)) + cwd = os.getcwd() + os.chdir(_BENCH_DIR) + try: + process_predictions(str(pred_path), str(scored_path)) + finally: + os.chdir(cwd) + return json.loads(scored_path.read_text()) + + +def aggregate(scored: list[dict]) -> dict: + """Macro-of-category-means: each category = mean of its per-sample scores, + overall = mean of the non-empty category means.""" + en: dict[str, list[float]] = defaultdict(list) + cn: dict[str, list[float]] = defaultdict(list) + for item in scored: + if "ignore" in item: + continue + t = item["type"] + if t in TYPE_TO_EN: + en[TYPE_TO_EN[t]].append(item["score"]) + elif t in TYPE_TO_CN: + cn[TYPE_TO_CN[t]].append(item["score"]) + + def cat_scores(buckets, categories): + return { + c: { + "avg": (sum(buckets[c]) / len(buckets[c]) if buckets[c] else 0.0), + "count": len(buckets[c]), + } + for c in categories + } + + en_scores = cat_scores(en, EN_CATEGORIES) + cn_scores = cat_scores(cn, CN_CATEGORIES) + en_avgs = [en_scores[c]["avg"] for c in EN_CATEGORIES if en_scores[c]["count"]] + cn_avgs = [cn_scores[c]["avg"] for c in CN_CATEGORIES if cn_scores[c]["count"]] + return { + "en_scores": en_scores, + "cn_scores": cn_scores, + "en_overall": sum(en_avgs) / len(en_avgs) if en_avgs else 0.0, + "cn_overall": sum(cn_avgs) / len(cn_avgs) if cn_avgs else 0.0, + } + + +def score(records: list[dict], samples: list[dict]) -> dict: + by_id = {s["id"]: s for s in samples} + preds = [] + for r in records: + s = by_id.get(r["id"]) + if s is None: + continue + preds.append( + { + "id": r["id"], + "dataset_name": s["dataset_name"], + "type": s["type"], + "question": s["question"], + "answers": s["answers"], + "predict": r.get("prediction") or "", + } + ) + scored = _run_per_sample_scorer(preds) + return aggregate(scored) diff --git a/benchmarks/ocrbench_v2/ocrbench_v2.py b/benchmarks/ocrbench_v2/ocrbench_v2.py deleted file mode 100644 index 58c8a10..0000000 --- a/benchmarks/ocrbench_v2/ocrbench_v2.py +++ /dev/null @@ -1,350 +0,0 @@ -""" -OCRBench v2 benchmark for Interfaze. -10,000 QA pairs across 30 task types (EN + CN). - -Usage: - uv run python benchmarks/OCRBench_v2/ocrbench_v2.py - uv run python benchmarks/OCRBench_v2/ocrbench_v2.py --predict-only - uv run python benchmarks/OCRBench_v2/ocrbench_v2.py --evaluate-only -""" - -import sys -import json -import asyncio -import argparse -import base64 -from pathlib import Path -from io import BytesIO - -from datasets import load_dataset -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -BENCHMARK_DIR = Path(__file__).resolve().parent -RESULTS_DIR = PROJECT_ROOT / "results" -PRED_OUTPUT = RESULTS_DIR / "ocrbench_v2_predictions.json" -EVAL_OUTPUT = RESULTS_DIR / "ocrbench_v2_scored.json" - -sys.path.insert(0, str(PROJECT_ROOT)) -from src.commons import invoke_interfaze # noqa: E402 - -RATE_LIMIT = 25 -MAX_RETRIES = 3 - -TEXT_SPOTTING_PROMPT_TEMPLATE = """Use OCR on this image to spot all text at {level}. The OCR tool returns each detected text region with its text content and four corner coordinates: top_left, top_right, bottom_left, bottom_right (each as an x,y pixel pair). - -Then use run code to write a Python script that takes those OCR results and: -1. For each text region, compute the axis-aligned bounding box from the four corners: - - x1 = min of all x coordinates (leftmost) - - y1 = min of all y coordinates (topmost) - - x2 = max of all x coordinates (rightmost) - - y2 = max of all y coordinates (bottommost) -2. Normalize each coordinate to the range 0-1000 by dividing by the image width (for x) or height (for y) and multiplying by 1000, then rounding to an integer. -3. Print the results as a Python list. - -Your final answer must be ONLY a Python list in this exact format, with no markdown, no code fences, no explanation: -[(x1, y1, x2, y2, "text"), (x1, y1, x2, y2, "text"), ...]""" - - -def get_spotting_prompt(original_question: str) -> str: - """Preserve word-level vs line-level from the original question.""" - if "line-level" in original_question: - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="line-level") - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="word-level") - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def pil_to_data_url(image) -> str: - buffer = BytesIO() - image.save(buffer, format="JPEG", quality=95) - b64 = base64.b64encode(buffer.getvalue()).decode("utf-8") - return f"data:image/jpeg;base64,{b64}" - - -def build_messages(question: str, image_url: str) -> list[dict]: - return [ - { - "role": "user", - "content": [ - {"type": "text", "text": question}, - {"type": "image_url", "image_url": {"url": image_url}}, - ], - } - ] - - -async def process_sample(sample_meta: dict, rate_limiter): - messages = build_messages(sample_meta["question"], sample_meta["image_url"]) - - for attempt in range(MAX_RETRIES): - await rate_limiter.acquire() - try: - response = await asyncio.to_thread(invoke_interfaze, messages) - return response.choices[0].message.content - except Exception as e: - if attempt < MAX_RETRIES - 1: - await asyncio.sleep(2**attempt) - else: - print( - f"Failed after {MAX_RETRIES} attempts for id={sample_meta['id']}: {e}" - ) - return "" - - -BATCH_SIZE = 100 - - -async def run_predictions(): - print("Loading OCRBench v2 from HuggingFace...") - dataset = load_dataset("lmms-lab/OCRBench-v2", split="test") - total = len(dataset) - print(f"Loaded {total} samples") - - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - - # Resume: load existing predictions and only re-run failures - existing = {} - if PRED_OUTPUT.exists(): - with open(PRED_OUTPUT) as f: - for item in json.load(f): - if item.get("predict", "") != "": - existing[item["id"]] = item - print(f"Resuming: {len(existing)} successful predictions found, skipping them") - - rate_limiter = RateLimiter(RATE_LIMIT) - output_data = {} - output_data.update(existing) - num_retried = 0 - - from tqdm import tqdm - - for batch_start in tqdm(range(0, total, BATCH_SIZE), desc="Batches"): - batch_end = min(batch_start + BATCH_SIZE, total) - batch = dataset[batch_start:batch_end] - - # Encode images to base64 and extract metadata, skip already-done samples - samples = [] - for i in range(len(batch["id"])): - sample_id = batch["id"][i] - if sample_id in existing: - continue - image_url = pil_to_data_url(batch["image"][i]) - question = batch["question"][i] - # Use custom prompt for text spotting EN - if batch["type"][i] == "text spotting en": - question = get_spotting_prompt(question) - samples.append( - { - "id": sample_id, - "dataset_name": batch["dataset_name"][i], - "type": batch["type"][i], - "question": question, - "answers": batch["answers"][i], - "image_url": image_url, - } - ) - del batch - - if not samples: - continue - - num_retried += len(samples) - tasks = [process_sample(s, rate_limiter) for s in samples] - predictions = await tqdm_asyncio.gather( - *tasks, desc=f"Predicting {batch_start}-{batch_end}", leave=False - ) - - for sample, pred in zip(samples, predictions): - output_data[sample["id"]] = { - "id": sample["id"], - "dataset_name": sample["dataset_name"], - "type": sample["type"], - "question": sample["question"], - "answers": sample["answers"], - "predict": pred, - } - - del samples, predictions - - # Sort by id and save - final_data = [output_data[i] for i in sorted(output_data.keys())] - with open(PRED_OUTPUT, "w", encoding="utf-8") as f: - json.dump(final_data, f, ensure_ascii=False, indent=2) - - num_failures = sum(1 for d in final_data if d.get("predict", "") == "") - print(f"\nPredictions saved to {PRED_OUTPUT}") - print( - f"Total: {total} | Retried: {num_retried} | Remaining failures: {num_failures}" - ) - - -def run_evaluation(): - if not PRED_OUTPUT.exists(): - print(f"No predictions found at {PRED_OUTPUT}") - print("Run with --predict-only first, or without flags to do both.") - sys.exit(1) - - eval_scripts_dir = BENCHMARK_DIR / "eval_scripts" - sys.path.insert(0, str(eval_scripts_dir)) - - # spotting_metric.py uses relative paths, so chdir to benchmark dir - import os - original_cwd = os.getcwd() - os.chdir(BENCHMARK_DIR) - - # Step 1: Score each sample using eval.py - print("Step 1: Scoring individual samples...") - from eval import process_predictions # noqa: E402 - - EVAL_OUTPUT.parent.mkdir(parents=True, exist_ok=True) - process_predictions(str(PRED_OUTPUT), str(EVAL_OUTPUT)) - - os.chdir(original_cwd) - print(f"Scored results saved to {EVAL_OUTPUT}") - - # Step 2: Compute overall metrics using get_score.py logic - print("\nStep 2: Computing overall metrics...") - with open(EVAL_OUTPUT) as f: - scored_data = json.load(f) - - en_scores = { - "text_recognition": [], - "text_detection": [], - "text_spotting": [], - "relationship_extraction": [], - "element_parsing": [], - "mathematical_calculation": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - cn_scores = { - "text_recognition": [], - "relationship_extraction": [], - "element_parsing": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - - type_to_en = { - "text recognition en": "text_recognition", - "fine-grained text recognition en": "text_recognition", - "full-page OCR en": "text_recognition", - "text grounding en": "text_detection", - "VQA with position en": "text_detection", - "text spotting en": "text_spotting", - "key information extraction en": "relationship_extraction", - "key information mapping en": "relationship_extraction", - "document parsing en": "element_parsing", - "chart parsing en": "element_parsing", - "table parsing en": "element_parsing", - "formula recognition en": "element_parsing", - "math QA en": "mathematical_calculation", - "text counting en": "mathematical_calculation", - "document classification en": "visual_text_understanding", - "cognition VQA en": "visual_text_understanding", - "diagram QA en": "visual_text_understanding", - "reasoning VQA en": "knowledge_reasoning", - "science QA en": "knowledge_reasoning", - "APP agent en": "knowledge_reasoning", - "ASCII art classification en": "knowledge_reasoning", - } - type_to_cn = { - "full-page OCR cn": "text_recognition", - "key information extraction cn": "relationship_extraction", - "handwritten answer extraction cn": "relationship_extraction", - "document parsing cn": "element_parsing", - "table parsing cn": "element_parsing", - "formula recognition cn": "element_parsing", - "cognition VQA cn": "visual_text_understanding", - "reasoning VQA cn": "knowledge_reasoning", - "text translation cn": "knowledge_reasoning", - } - - for item in scored_data: - if "ignore" in item: - continue - t = item["type"] - if t in type_to_en: - en_scores[type_to_en[t]].append(item["score"]) - elif t in type_to_cn: - cn_scores[type_to_cn[t]].append(item["score"]) - - def avg(lst): - return sum(lst) / len(lst) if lst else 0.0 - - en_avgs = {k: avg(v) for k, v in en_scores.items() if v} - cn_avgs = {k: avg(v) for k, v in cn_scores.items() if v} - en_overall = avg(list(en_avgs.values())) - cn_overall = avg(list(cn_avgs.values())) - - print(f"\n{'=' * 60}") - print("OCRBench v2 Results (Interfaze)") - print(f"{'=' * 60}") - print(f"\n{'Category':<30} {'EN':>8} {'CN':>8}") - print("-" * 48) - all_cats = sorted(set(list(en_scores.keys()) + list(cn_scores.keys()))) - for cat in all_cats: - en_val = f"{en_avgs[cat]:.3f}" if cat in en_avgs else " -" - cn_val = f"{cn_avgs[cat]:.3f}" if cat in cn_avgs else " -" - print(f"{cat:<30} {en_val:>8} {cn_val:>8}") - print("-" * 48) - print(f"{'OVERALL':<30} {en_overall:>8.3f} {cn_overall:>8.3f}") - - # Save metrics - metrics = { - "en_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in en_scores.items() - }, - "cn_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in cn_scores.items() - }, - "en_overall": en_overall, - "cn_overall": cn_overall, - "model": "interfaze-beta", - } - metrics_path = RESULTS_DIR / "ocrbench_v2_metrics.json" - with open(metrics_path, "w") as f: - json.dump(metrics, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - parser = argparse.ArgumentParser(description="OCRBench v2 benchmark for Interfaze") - parser.add_argument( - "--predict-only", action="store_true", help="Only generate predictions" - ) - parser.add_argument( - "--evaluate-only", action="store_true", help="Only run evaluation" - ) - args = parser.parse_args() - - if args.evaluate_only: - run_evaluation() - elif args.predict_only: - asyncio.run(run_predictions()) - else: - asyncio.run(run_predictions()) - run_evaluation() - - -if __name__ == "__main__": - main() diff --git a/benchmarks/ocrbench_v2/ocrbench_v2_anthropic.py b/benchmarks/ocrbench_v2/ocrbench_v2_anthropic.py deleted file mode 100644 index 0527640..0000000 --- a/benchmarks/ocrbench_v2/ocrbench_v2_anthropic.py +++ /dev/null @@ -1,355 +0,0 @@ -""" -OCRBench v2 benchmark for Anthropic Claude. -10,000 QA pairs across 30 task types (EN + CN). - -Usage: - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_anthropic - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_anthropic --predict-only - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_anthropic --evaluate-only -""" - -import sys -import json -import asyncio -import argparse -import base64 -from pathlib import Path -from io import BytesIO - -from datasets import load_dataset -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -BENCHMARK_DIR = Path(__file__).resolve().parent -RESULTS_DIR = PROJECT_ROOT / "results" -PRED_OUTPUT = RESULTS_DIR / "ocrbench_v2_anthropic_predictions.json" -EVAL_OUTPUT = RESULTS_DIR / "ocrbench_v2_anthropic_scored.json" - -sys.path.insert(0, str(PROJECT_ROOT)) -from src.commons_anthropic import invoke_anthropic # noqa: E402 - -MODEL = "claude-sonnet-4-6" -RATE_LIMIT = 25 -MAX_RETRIES = 3 - -TEXT_SPOTTING_PROMPT_TEMPLATE = """Use OCR on this image to spot all text at {level}. The OCR tool returns each detected text region with its text content and four corner coordinates: top_left, top_right, bottom_left, bottom_right (each as an x,y pixel pair). - -Then use run code to write a Python script that takes those OCR results and: -1. For each text region, compute the axis-aligned bounding box from the four corners: - - x1 = min of all x coordinates (leftmost) - - y1 = min of all y coordinates (topmost) - - x2 = max of all x coordinates (rightmost) - - y2 = max of all y coordinates (bottommost) -2. Normalize each coordinate to the range 0-1000 by dividing by the image width (for x) or height (for y) and multiplying by 1000, then rounding to an integer. -3. Print the results as a Python list. - -Your final answer must be ONLY a Python list in this exact format, with no markdown, no code fences, no explanation: -[(x1, y1, x2, y2, "text"), (x1, y1, x2, y2, "text"), ...]""" - - -def get_spotting_prompt(original_question: str) -> str: - if "line-level" in original_question: - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="line-level") - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="word-level") - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def pil_to_b64(image) -> str: - buffer = BytesIO() - image.save(buffer, format="JPEG", quality=95) - return base64.b64encode(buffer.getvalue()).decode("utf-8") - - -def build_messages(question: str, image_b64: str) -> list[dict]: - return [ - { - "role": "user", - "content": [ - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/jpeg", - "data": image_b64, - }, - }, - {"type": "text", "text": question}, - ], - } - ] - - -def extract_text(response) -> str: - for block in response.content: - if block.type == "text": - return block.text - return "" - - -async def process_sample(sample_meta: dict, rate_limiter): - messages = build_messages(sample_meta["question"], sample_meta["image_b64"]) - - for attempt in range(MAX_RETRIES): - await rate_limiter.acquire() - try: - response = await asyncio.to_thread(invoke_anthropic, messages, MODEL) - return extract_text(response) - except Exception as e: - if attempt < MAX_RETRIES - 1: - await asyncio.sleep(2**attempt) - else: - print( - f"Failed after {MAX_RETRIES} attempts for id={sample_meta['id']}: {e}" - ) - return "" - - -BATCH_SIZE = 100 - - -async def run_predictions(): - print("Loading OCRBench v2 from HuggingFace...") - dataset = load_dataset("lmms-lab/OCRBench-v2", split="test") - total = len(dataset) - print(f"Loaded {total} samples") - - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - - existing = {} - if PRED_OUTPUT.exists(): - with open(PRED_OUTPUT) as f: - for item in json.load(f): - if item.get("predict", "") != "": - existing[item["id"]] = item - print(f"Resuming: {len(existing)} successful predictions found, skipping them") - - rate_limiter = RateLimiter(RATE_LIMIT) - output_data = {} - output_data.update(existing) - num_retried = 0 - - from tqdm import tqdm - - for batch_start in tqdm(range(0, total, BATCH_SIZE), desc="Batches"): - batch_end = min(batch_start + BATCH_SIZE, total) - batch = dataset[batch_start:batch_end] - - samples = [] - for i in range(len(batch["id"])): - sample_id = batch["id"][i] - if sample_id in existing: - continue - image_b64 = pil_to_b64(batch["image"][i]) - question = batch["question"][i] - if batch["type"][i] == "text spotting en": - question = get_spotting_prompt(question) - samples.append( - { - "id": sample_id, - "dataset_name": batch["dataset_name"][i], - "type": batch["type"][i], - "question": question, - "answers": batch["answers"][i], - "image_b64": image_b64, - } - ) - del batch - - if not samples: - continue - - num_retried += len(samples) - tasks = [process_sample(s, rate_limiter) for s in samples] - predictions = await tqdm_asyncio.gather( - *tasks, desc=f"Predicting {batch_start}-{batch_end}", leave=False - ) - - for sample, pred in zip(samples, predictions): - output_data[sample["id"]] = { - "id": sample["id"], - "dataset_name": sample["dataset_name"], - "type": sample["type"], - "question": sample["question"], - "answers": sample["answers"], - "predict": pred, - } - - del samples, predictions - - final_data = [output_data[i] for i in sorted(output_data.keys())] - with open(PRED_OUTPUT, "w", encoding="utf-8") as f: - json.dump(final_data, f, ensure_ascii=False, indent=2) - - num_failures = sum(1 for d in final_data if d.get("predict", "") == "") - print(f"\nPredictions saved to {PRED_OUTPUT}") - print( - f"Total: {total} | Retried: {num_retried} | Remaining failures: {num_failures}" - ) - - -def run_evaluation(): - if not PRED_OUTPUT.exists(): - print(f"No predictions found at {PRED_OUTPUT}") - print("Run with --predict-only first, or without flags to do both.") - sys.exit(1) - - eval_scripts_dir = BENCHMARK_DIR / "eval_scripts" - sys.path.insert(0, str(eval_scripts_dir)) - - import os - original_cwd = os.getcwd() - os.chdir(BENCHMARK_DIR) - - print("Step 1: Scoring individual samples...") - from benchmarks.ocrbench_v2.eval_scripts.eval import process_predictions # noqa: E402 - - EVAL_OUTPUT.parent.mkdir(parents=True, exist_ok=True) - process_predictions(str(PRED_OUTPUT), str(EVAL_OUTPUT)) - - os.chdir(original_cwd) - print(f"Scored results saved to {EVAL_OUTPUT}") - - print("\nStep 2: Computing overall metrics...") - with open(EVAL_OUTPUT) as f: - scored_data = json.load(f) - - en_scores = { - "text_recognition": [], - "text_detection": [], - "text_spotting": [], - "relationship_extraction": [], - "element_parsing": [], - "mathematical_calculation": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - cn_scores = { - "text_recognition": [], - "relationship_extraction": [], - "element_parsing": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - - type_to_en = { - "text recognition en": "text_recognition", - "fine-grained text recognition en": "text_recognition", - "full-page OCR en": "text_recognition", - "text grounding en": "text_detection", - "VQA with position en": "text_detection", - "text spotting en": "text_spotting", - "key information extraction en": "relationship_extraction", - "key information mapping en": "relationship_extraction", - "document parsing en": "element_parsing", - "chart parsing en": "element_parsing", - "table parsing en": "element_parsing", - "formula recognition en": "element_parsing", - "math QA en": "mathematical_calculation", - "text counting en": "mathematical_calculation", - "document classification en": "visual_text_understanding", - "cognition VQA en": "visual_text_understanding", - "diagram QA en": "visual_text_understanding", - "reasoning VQA en": "knowledge_reasoning", - "science QA en": "knowledge_reasoning", - "APP agent en": "knowledge_reasoning", - "ASCII art classification en": "knowledge_reasoning", - } - type_to_cn = { - "full-page OCR cn": "text_recognition", - "key information extraction cn": "relationship_extraction", - "handwritten answer extraction cn": "relationship_extraction", - "document parsing cn": "element_parsing", - "table parsing cn": "element_parsing", - "formula recognition cn": "element_parsing", - "cognition VQA cn": "visual_text_understanding", - "reasoning VQA cn": "knowledge_reasoning", - "text translation cn": "knowledge_reasoning", - } - - for item in scored_data: - if "ignore" in item: - continue - t = item["type"] - if t in type_to_en: - en_scores[type_to_en[t]].append(item["score"]) - elif t in type_to_cn: - cn_scores[type_to_cn[t]].append(item["score"]) - - def avg(lst): - return sum(lst) / len(lst) if lst else 0.0 - - en_avgs = {k: avg(v) for k, v in en_scores.items() if v} - cn_avgs = {k: avg(v) for k, v in cn_scores.items() if v} - en_overall = avg(list(en_avgs.values())) - cn_overall = avg(list(cn_avgs.values())) - - print(f"\n{'=' * 60}") - print(f"OCRBench v2 Results ({MODEL})") - print(f"{'=' * 60}") - print(f"\n{'Category':<30} {'EN':>8} {'CN':>8}") - print("-" * 48) - all_cats = sorted(set(list(en_scores.keys()) + list(cn_scores.keys()))) - for cat in all_cats: - en_val = f"{en_avgs[cat]:.3f}" if cat in en_avgs else " -" - cn_val = f"{cn_avgs[cat]:.3f}" if cat in cn_avgs else " -" - print(f"{cat:<30} {en_val:>8} {cn_val:>8}") - print("-" * 48) - print(f"{'OVERALL':<30} {en_overall:>8.3f} {cn_overall:>8.3f}") - - metrics = { - "en_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in en_scores.items() - }, - "cn_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in cn_scores.items() - }, - "en_overall": en_overall, - "cn_overall": cn_overall, - "model": MODEL, - } - metrics_path = RESULTS_DIR / "ocrbench_v2_anthropic_metrics.json" - with open(metrics_path, "w") as f: - json.dump(metrics, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - parser = argparse.ArgumentParser(description="OCRBench v2 benchmark for Anthropic Claude") - parser.add_argument( - "--predict-only", action="store_true", help="Only generate predictions" - ) - parser.add_argument( - "--evaluate-only", action="store_true", help="Only run evaluation" - ) - args = parser.parse_args() - - if args.evaluate_only: - run_evaluation() - elif args.predict_only: - asyncio.run(run_predictions()) - else: - asyncio.run(run_predictions()) - run_evaluation() - - -if __name__ == "__main__": - main() diff --git a/benchmarks/ocrbench_v2/ocrbench_v2_fireworks.py b/benchmarks/ocrbench_v2/ocrbench_v2_fireworks.py deleted file mode 100644 index e63d37d..0000000 --- a/benchmarks/ocrbench_v2/ocrbench_v2_fireworks.py +++ /dev/null @@ -1,441 +0,0 @@ -""" -OCRBench v2 benchmark for Fireworks-hosted models (default: Inkling). -10,000 QA pairs across 30 task types (EN + CN). - -Inkling always thinks; reasoning_effort="none" is its floor, not a true off. - -Usage: - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks --model accounts/fireworks/models/inkling-small - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks --predict-only - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks --evaluate-only -""" - -import os -import re -import sys -import json -import asyncio -import argparse -import base64 -from pathlib import Path -from io import BytesIO - -from datasets import load_dataset -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -BENCHMARK_DIR = Path(__file__).resolve().parent -RESULTS_DIR = PROJECT_ROOT / "results" - -sys.path.insert(0, str(PROJECT_ROOT)) -from src.commons_fireworks import INKLING, fireworks_client # noqa: E402 - -# Provider is selectable because the same model can be reachable on more than one -# host, and the hosts are NOT equivalent: on OpenRouter reasoning effort "none" -# truly disables thinking, while on Fireworks it is only a floor. -PROVIDER = "fireworks" -CLIENT = fireworks_client - - -def set_provider(provider: str) -> None: - global PROVIDER, CLIENT - PROVIDER = provider - if provider == "openrouter": - from src.commons_openrouter import openrouter_client - - CLIENT = openrouter_client - else: - CLIENT = fireworks_client - - -def _reasoning_kwargs() -> dict: - if PROVIDER == "openrouter": - return {"extra_body": {"reasoning": {"effort": REASONING_EFFORT}}} - return {"reasoning_effort": REASONING_EFFORT} - -MODEL = INKLING -REASONING_EFFORT = "none" -RATE_LIMIT = 25 -MAX_RETRIES = 3 - -# Cap on requests in flight. RATE_LIMIT bounds how many requests *start* per -# second but not how many are outstanding — a whole batch launched at once -# queues server-side and eventually 429s after minutes of latency. Override -# with BENCH_MAX_IN_FLIGHT. -MAX_IN_FLIGHT = int(os.getenv("BENCH_MAX_IN_FLIGHT", "8")) - -# Set from MODEL in main() so inkling and inkling-small keep separate -# checkpoints and can run side by side. -TAG = "ocrbench_v2_fireworks_inkling" -PRED_OUTPUT = RESULTS_DIR / f"{TAG}_predictions.json" -EVAL_OUTPUT = RESULTS_DIR / f"{TAG}_scored.json" -METRICS_OUTPUT = RESULTS_DIR / f"{TAG}_metrics.json" - -TEXT_SPOTTING_PROMPT_TEMPLATE = """Use OCR on this image to spot all text at {level}. The OCR tool returns each detected text region with its text content and four corner coordinates: top_left, top_right, bottom_left, bottom_right (each as an x,y pixel pair). - -Then use run code to write a Python script that takes those OCR results and: -1. For each text region, compute the axis-aligned bounding box from the four corners: - - x1 = min of all x coordinates (leftmost) - - y1 = min of all y coordinates (topmost) - - x2 = max of all x coordinates (rightmost) - - y2 = max of all y coordinates (bottommost) -2. Normalize each coordinate to the range 0-1000 by dividing by the image width (for x) or height (for y) and multiplying by 1000, then rounding to an integer. -3. Print the results as a Python list. - -Your final answer must be ONLY a Python list in this exact format, with no markdown, no code fences, no explanation: -[(x1, y1, x2, y2, "text"), (x1, y1, x2, y2, "text"), ...]""" - - -def model_slug(model: str) -> str: - """accounts/fireworks/models/inkling-small -> inkling-small""" - return re.sub(r"[^a-z0-9]+", "-", model.rsplit("/", 1)[-1].lower()).strip("-") - - -def set_paths(model: str) -> None: - global TAG, PRED_OUTPUT, EVAL_OUTPUT, METRICS_OUTPUT - TAG = f"ocrbench_v2_fireworks_{model_slug(model)}" - PRED_OUTPUT = RESULTS_DIR / f"{TAG}_predictions.json" - EVAL_OUTPUT = RESULTS_DIR / f"{TAG}_scored.json" - METRICS_OUTPUT = RESULTS_DIR / f"{TAG}_metrics.json" - - -def get_spotting_prompt(original_question: str) -> str: - if "line-level" in original_question: - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="line-level") - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="word-level") - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def pil_to_data_url(image) -> str: - buffer = BytesIO() - if image.mode != "RGB": - image = image.convert("RGB") - image.save(buffer, format="JPEG", quality=95) - b64 = base64.b64encode(buffer.getvalue()).decode("utf-8") - return f"data:image/jpeg;base64,{b64}" - - -def build_messages(question: str, image_url: str) -> list[dict]: - return [ - { - "role": "user", - "content": [ - {"type": "text", "text": question}, - {"type": "image_url", "image_url": {"url": image_url}}, - ], - } - ] - - -async def process_sample(sample_meta: dict, rate_limiter, sem: asyncio.Semaphore): - async with sem: - return await _process_sample(sample_meta, rate_limiter) - - -async def _process_sample(sample_meta: dict, rate_limiter): - messages = build_messages(sample_meta["question"], sample_meta["image_url"]) - - for attempt in range(MAX_RETRIES): - await rate_limiter.acquire() - try: - response = await asyncio.to_thread( - CLIENT.chat.completions.create, - model=MODEL, - messages=messages, - **_reasoning_kwargs(), - ) - return response.choices[0].message.content or "" - except Exception as e: - if attempt < MAX_RETRIES - 1: - await asyncio.sleep(2**attempt) - else: - print( - f"Failed after {MAX_RETRIES} attempts for id={sample_meta['id']}: {e}" - ) - return "" - - -BATCH_SIZE = 200 - - -async def run_predictions(limit: int | None = None): - print("Loading OCRBench v2 from HuggingFace...") - dataset = load_dataset("lmms-lab/OCRBench-v2", split="test") - if limit is not None: - dataset = dataset.select(range(min(limit, len(dataset)))) - print(f"--limit applied: first {len(dataset)} sample(s) only") - total = len(dataset) - print(f"Loaded {total} samples") - - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - - existing = {} - if PRED_OUTPUT.exists(): - with open(PRED_OUTPUT) as f: - for item in json.load(f): - if item.get("predict", "") != "": - existing[item["id"]] = item - print(f"Resuming: {len(existing)} successful predictions found, skipping them") - - rate_limiter = RateLimiter(RATE_LIMIT) - sem = asyncio.Semaphore(MAX_IN_FLIGHT) - output_data = {} - output_data.update(existing) - num_retried = 0 - - from tqdm import tqdm - - for batch_start in tqdm(range(0, total, BATCH_SIZE), desc="Batches"): - batch_end = min(batch_start + BATCH_SIZE, total) - batch = dataset[batch_start:batch_end] - - samples = [] - for i in range(len(batch["id"])): - sample_id = batch["id"][i] - if sample_id in existing: - continue - image_url = pil_to_data_url(batch["image"][i]) - question = batch["question"][i] - if batch["type"][i] == "text spotting en": - question = get_spotting_prompt(question) - samples.append( - { - "id": sample_id, - "dataset_name": batch["dataset_name"][i], - "type": batch["type"][i], - "question": question, - "answers": batch["answers"][i], - "image_url": image_url, - } - ) - del batch - - if not samples: - continue - - num_retried += len(samples) - tasks = [process_sample(s, rate_limiter, sem) for s in samples] - predictions = await tqdm_asyncio.gather( - *tasks, desc=f"Predicting {batch_start}-{batch_end}", leave=False - ) - - for sample, pred in zip(samples, predictions): - output_data[sample["id"]] = { - "id": sample["id"], - "dataset_name": sample["dataset_name"], - "type": sample["type"], - "question": sample["question"], - "answers": sample["answers"], - "predict": pred, - } - - del samples, predictions - - # Checkpoint after every batch — a 10k-sample run is too long to lose. - final_data = [output_data[i] for i in sorted(output_data.keys())] - with open(PRED_OUTPUT, "w", encoding="utf-8") as f: - json.dump(final_data, f, ensure_ascii=False, indent=2) - - final_data = [output_data[i] for i in sorted(output_data.keys())] - with open(PRED_OUTPUT, "w", encoding="utf-8") as f: - json.dump(final_data, f, ensure_ascii=False, indent=2) - - num_failures = sum(1 for d in final_data if d.get("predict", "") == "") - print(f"\nPredictions saved to {PRED_OUTPUT}") - print( - f"Total: {total} | Retried: {num_retried} | Remaining failures: {num_failures}" - ) - - -def run_evaluation(): - if not PRED_OUTPUT.exists(): - print(f"No predictions found at {PRED_OUTPUT}") - print("Run with --predict-only first, or without flags to do both.") - sys.exit(1) - - eval_scripts_dir = BENCHMARK_DIR / "eval_scripts" - sys.path.insert(0, str(eval_scripts_dir)) - - import os - - original_cwd = os.getcwd() - os.chdir(BENCHMARK_DIR) - - print("Step 1: Scoring individual samples...") - from benchmarks.ocrbench_v2.eval_scripts.eval import process_predictions # noqa: E402 - - EVAL_OUTPUT.parent.mkdir(parents=True, exist_ok=True) - process_predictions(str(PRED_OUTPUT), str(EVAL_OUTPUT)) - - os.chdir(original_cwd) - print(f"Scored results saved to {EVAL_OUTPUT}") - - print("\nStep 2: Computing overall metrics...") - with open(EVAL_OUTPUT) as f: - scored_data = json.load(f) - - en_scores = { - "text_recognition": [], - "text_detection": [], - "text_spotting": [], - "relationship_extraction": [], - "element_parsing": [], - "mathematical_calculation": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - cn_scores = { - "text_recognition": [], - "relationship_extraction": [], - "element_parsing": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - - type_to_en = { - "text recognition en": "text_recognition", - "fine-grained text recognition en": "text_recognition", - "full-page OCR en": "text_recognition", - "text grounding en": "text_detection", - "VQA with position en": "text_detection", - "text spotting en": "text_spotting", - "key information extraction en": "relationship_extraction", - "key information mapping en": "relationship_extraction", - "document parsing en": "element_parsing", - "chart parsing en": "element_parsing", - "table parsing en": "element_parsing", - "formula recognition en": "element_parsing", - "math QA en": "mathematical_calculation", - "text counting en": "mathematical_calculation", - "document classification en": "visual_text_understanding", - "cognition VQA en": "visual_text_understanding", - "diagram QA en": "visual_text_understanding", - "reasoning VQA en": "knowledge_reasoning", - "science QA en": "knowledge_reasoning", - "APP agent en": "knowledge_reasoning", - "ASCII art classification en": "knowledge_reasoning", - } - type_to_cn = { - "full-page OCR cn": "text_recognition", - "key information extraction cn": "relationship_extraction", - "handwritten answer extraction cn": "relationship_extraction", - "document parsing cn": "element_parsing", - "table parsing cn": "element_parsing", - "formula recognition cn": "element_parsing", - "cognition VQA cn": "visual_text_understanding", - "reasoning VQA cn": "knowledge_reasoning", - "text translation cn": "knowledge_reasoning", - } - - for item in scored_data: - if "ignore" in item: - continue - t = item["type"] - if t in type_to_en: - en_scores[type_to_en[t]].append(item["score"]) - elif t in type_to_cn: - cn_scores[type_to_cn[t]].append(item["score"]) - - def avg(lst): - return sum(lst) / len(lst) if lst else 0.0 - - en_avgs = {k: avg(v) for k, v in en_scores.items() if v} - cn_avgs = {k: avg(v) for k, v in cn_scores.items() if v} - en_overall = avg(list(en_avgs.values())) - cn_overall = avg(list(cn_avgs.values())) - - print(f"\n{'=' * 60}") - print(f"OCRBench v2 Results ({MODEL})") - print(f"{'=' * 60}") - print(f"\n{'Category':<30} {'EN':>8} {'CN':>8}") - print("-" * 48) - all_cats = sorted(set(list(en_scores.keys()) + list(cn_scores.keys()))) - for cat in all_cats: - en_val = f"{en_avgs[cat]:.3f}" if cat in en_avgs else " -" - cn_val = f"{cn_avgs[cat]:.3f}" if cat in cn_avgs else " -" - print(f"{cat:<30} {en_val:>8} {cn_val:>8}") - print("-" * 48) - print(f"{'OVERALL':<30} {en_overall:>8.3f} {cn_overall:>8.3f}") - - metrics = { - "en_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in en_scores.items() - }, - "cn_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in cn_scores.items() - }, - "en_overall": en_overall, - "cn_overall": cn_overall, - "provider": "fireworks", - "model": MODEL, - "reasoning_effort": REASONING_EFFORT, - } - with open(METRICS_OUTPUT, "w") as f: - json.dump(metrics, f, indent=2) - print(f"\nMetrics saved to {METRICS_OUTPUT}") - - -def main(): - global MODEL, REASONING_EFFORT - parser = argparse.ArgumentParser( - description="OCRBench v2 benchmark for Fireworks-hosted models (Inkling)" - ) - parser.add_argument( - "--model", default=INKLING, help="Model id for the chosen provider" - ) - parser.add_argument( - "--provider", default="fireworks", choices=["fireworks", "openrouter"] - ) - parser.add_argument( - "--reasoning", default="none", help="reasoning effort passed to the provider" - ) - parser.add_argument( - "--predict-only", action="store_true", help="Only generate predictions" - ) - parser.add_argument( - "--evaluate-only", action="store_true", help="Only run evaluation" - ) - parser.add_argument( - "--limit", - type=int, - default=None, - help="Only predict the first N dataset rows (smoke test)", - ) - args = parser.parse_args() - - MODEL = args.model - REASONING_EFFORT = args.reasoning - set_provider(args.provider) - set_paths(MODEL) - - if args.evaluate_only: - run_evaluation() - elif args.predict_only: - asyncio.run(run_predictions(limit=args.limit)) - else: - asyncio.run(run_predictions(limit=args.limit)) - run_evaluation() - - -if __name__ == "__main__": - main() diff --git a/benchmarks/ocrbench_v2/ocrbench_v2_gemini.py b/benchmarks/ocrbench_v2/ocrbench_v2_gemini.py deleted file mode 100644 index b862955..0000000 --- a/benchmarks/ocrbench_v2/ocrbench_v2_gemini.py +++ /dev/null @@ -1,346 +0,0 @@ -""" -OCRBench v2 benchmark for Google Gemini. -10,000 QA pairs across 30 task types (EN + CN). - -Usage: - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_gemini - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_gemini --predict-only - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_gemini --evaluate-only -""" - -import sys -import json -import asyncio -import argparse -from pathlib import Path -from io import BytesIO - -from datasets import load_dataset -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -BENCHMARK_DIR = Path(__file__).resolve().parent -RESULTS_DIR = PROJECT_ROOT / "results" -PRED_OUTPUT = RESULTS_DIR / "ocrbench_v2_gemini_predictions.json" -EVAL_OUTPUT = RESULTS_DIR / "ocrbench_v2_gemini_scored.json" - -sys.path.insert(0, str(PROJECT_ROOT)) -from src.commons_gemini import invoke_gemini # noqa: E402 - -MODEL = "gemini-3-flash-preview" -RATE_LIMIT = 25 -MAX_RETRIES = 3 - -TEXT_SPOTTING_PROMPT_TEMPLATE = """Use OCR on this image to spot all text at {level}. The OCR tool returns each detected text region with its text content and four corner coordinates: top_left, top_right, bottom_left, bottom_right (each as an x,y pixel pair). - -Then use run code to write a Python script that takes those OCR results and: -1. For each text region, compute the axis-aligned bounding box from the four corners: - - x1 = min of all x coordinates (leftmost) - - y1 = min of all y coordinates (topmost) - - x2 = max of all x coordinates (rightmost) - - y2 = max of all y coordinates (bottommost) -2. Normalize each coordinate to the range 0-1000 by dividing by the image width (for x) or height (for y) and multiplying by 1000, then rounding to an integer. -3. Print the results as a Python list. - -Your final answer must be ONLY a Python list in this exact format, with no markdown, no code fences, no explanation: -[(x1, y1, x2, y2, "text"), (x1, y1, x2, y2, "text"), ...]""" - - -def get_spotting_prompt(original_question: str) -> str: - if "line-level" in original_question: - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="line-level") - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="word-level") - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def pil_to_jpeg_bytes(image) -> bytes: - buffer = BytesIO() - image.save(buffer, format="JPEG", quality=95) - return buffer.getvalue() - - -def build_contents(question: str, image_bytes: bytes) -> list: - from google.genai import types - - return [ - types.Part.from_bytes(data=image_bytes, mime_type="image/jpeg"), - question, - ] - - -def extract_text(response) -> str: - text = getattr(response, "text", None) - return text or "" - - -async def process_sample(sample_meta: dict, rate_limiter): - contents = build_contents(sample_meta["question"], sample_meta["image_bytes"]) - - for attempt in range(MAX_RETRIES): - await rate_limiter.acquire() - try: - response = await asyncio.to_thread(invoke_gemini, contents, MODEL) - return extract_text(response) - except Exception as e: - if attempt < MAX_RETRIES - 1: - await asyncio.sleep(2**attempt) - else: - print( - f"Failed after {MAX_RETRIES} attempts for id={sample_meta['id']}: {e}" - ) - return "" - - -BATCH_SIZE = 100 - - -async def run_predictions(): - print("Loading OCRBench v2 from HuggingFace...") - dataset = load_dataset("lmms-lab/OCRBench-v2", split="test") - total = len(dataset) - print(f"Loaded {total} samples") - - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - - existing = {} - if PRED_OUTPUT.exists(): - with open(PRED_OUTPUT) as f: - for item in json.load(f): - if item.get("predict", "") != "": - existing[item["id"]] = item - print(f"Resuming: {len(existing)} successful predictions found, skipping them") - - rate_limiter = RateLimiter(RATE_LIMIT) - output_data = {} - output_data.update(existing) - num_retried = 0 - - from tqdm import tqdm - - for batch_start in tqdm(range(0, total, BATCH_SIZE), desc="Batches"): - batch_end = min(batch_start + BATCH_SIZE, total) - batch = dataset[batch_start:batch_end] - - samples = [] - for i in range(len(batch["id"])): - sample_id = batch["id"][i] - if sample_id in existing: - continue - image_bytes = pil_to_jpeg_bytes(batch["image"][i]) - question = batch["question"][i] - if batch["type"][i] == "text spotting en": - question = get_spotting_prompt(question) - samples.append( - { - "id": sample_id, - "dataset_name": batch["dataset_name"][i], - "type": batch["type"][i], - "question": question, - "answers": batch["answers"][i], - "image_bytes": image_bytes, - } - ) - del batch - - if not samples: - continue - - num_retried += len(samples) - tasks = [process_sample(s, rate_limiter) for s in samples] - predictions = await tqdm_asyncio.gather( - *tasks, desc=f"Predicting {batch_start}-{batch_end}", leave=False - ) - - for sample, pred in zip(samples, predictions): - output_data[sample["id"]] = { - "id": sample["id"], - "dataset_name": sample["dataset_name"], - "type": sample["type"], - "question": sample["question"], - "answers": sample["answers"], - "predict": pred, - } - - del samples, predictions - - final_data = [output_data[i] for i in sorted(output_data.keys())] - with open(PRED_OUTPUT, "w", encoding="utf-8") as f: - json.dump(final_data, f, ensure_ascii=False, indent=2) - - num_failures = sum(1 for d in final_data if d.get("predict", "") == "") - print(f"\nPredictions saved to {PRED_OUTPUT}") - print( - f"Total: {total} | Retried: {num_retried} | Remaining failures: {num_failures}" - ) - - -def run_evaluation(): - if not PRED_OUTPUT.exists(): - print(f"No predictions found at {PRED_OUTPUT}") - print("Run with --predict-only first, or without flags to do both.") - sys.exit(1) - - eval_scripts_dir = BENCHMARK_DIR / "eval_scripts" - sys.path.insert(0, str(eval_scripts_dir)) - - import os - original_cwd = os.getcwd() - os.chdir(BENCHMARK_DIR) - - print("Step 1: Scoring individual samples...") - from benchmarks.ocrbench_v2.eval_scripts.eval import process_predictions # noqa: E402 - - EVAL_OUTPUT.parent.mkdir(parents=True, exist_ok=True) - process_predictions(str(PRED_OUTPUT), str(EVAL_OUTPUT)) - - os.chdir(original_cwd) - print(f"Scored results saved to {EVAL_OUTPUT}") - - print("\nStep 2: Computing overall metrics...") - with open(EVAL_OUTPUT) as f: - scored_data = json.load(f) - - en_scores = { - "text_recognition": [], - "text_detection": [], - "text_spotting": [], - "relationship_extraction": [], - "element_parsing": [], - "mathematical_calculation": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - cn_scores = { - "text_recognition": [], - "relationship_extraction": [], - "element_parsing": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - - type_to_en = { - "text recognition en": "text_recognition", - "fine-grained text recognition en": "text_recognition", - "full-page OCR en": "text_recognition", - "text grounding en": "text_detection", - "VQA with position en": "text_detection", - "text spotting en": "text_spotting", - "key information extraction en": "relationship_extraction", - "key information mapping en": "relationship_extraction", - "document parsing en": "element_parsing", - "chart parsing en": "element_parsing", - "table parsing en": "element_parsing", - "formula recognition en": "element_parsing", - "math QA en": "mathematical_calculation", - "text counting en": "mathematical_calculation", - "document classification en": "visual_text_understanding", - "cognition VQA en": "visual_text_understanding", - "diagram QA en": "visual_text_understanding", - "reasoning VQA en": "knowledge_reasoning", - "science QA en": "knowledge_reasoning", - "APP agent en": "knowledge_reasoning", - "ASCII art classification en": "knowledge_reasoning", - } - type_to_cn = { - "full-page OCR cn": "text_recognition", - "key information extraction cn": "relationship_extraction", - "handwritten answer extraction cn": "relationship_extraction", - "document parsing cn": "element_parsing", - "table parsing cn": "element_parsing", - "formula recognition cn": "element_parsing", - "cognition VQA cn": "visual_text_understanding", - "reasoning VQA cn": "knowledge_reasoning", - "text translation cn": "knowledge_reasoning", - } - - for item in scored_data: - if "ignore" in item: - continue - t = item["type"] - if t in type_to_en: - en_scores[type_to_en[t]].append(item["score"]) - elif t in type_to_cn: - cn_scores[type_to_cn[t]].append(item["score"]) - - def avg(lst): - return sum(lst) / len(lst) if lst else 0.0 - - en_avgs = {k: avg(v) for k, v in en_scores.items() if v} - cn_avgs = {k: avg(v) for k, v in cn_scores.items() if v} - en_overall = avg(list(en_avgs.values())) - cn_overall = avg(list(cn_avgs.values())) - - print(f"\n{'=' * 60}") - print(f"OCRBench v2 Results ({MODEL})") - print(f"{'=' * 60}") - print(f"\n{'Category':<30} {'EN':>8} {'CN':>8}") - print("-" * 48) - all_cats = sorted(set(list(en_scores.keys()) + list(cn_scores.keys()))) - for cat in all_cats: - en_val = f"{en_avgs[cat]:.3f}" if cat in en_avgs else " -" - cn_val = f"{cn_avgs[cat]:.3f}" if cat in cn_avgs else " -" - print(f"{cat:<30} {en_val:>8} {cn_val:>8}") - print("-" * 48) - print(f"{'OVERALL':<30} {en_overall:>8.3f} {cn_overall:>8.3f}") - - metrics = { - "en_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in en_scores.items() - }, - "cn_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in cn_scores.items() - }, - "en_overall": en_overall, - "cn_overall": cn_overall, - "model": MODEL, - } - metrics_path = RESULTS_DIR / "ocrbench_v2_gemini_metrics.json" - with open(metrics_path, "w") as f: - json.dump(metrics, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - parser = argparse.ArgumentParser(description="OCRBench v2 benchmark for Google Gemini") - parser.add_argument( - "--predict-only", action="store_true", help="Only generate predictions" - ) - parser.add_argument( - "--evaluate-only", action="store_true", help="Only run evaluation" - ) - parser.add_argument( - "--model", default=MODEL, help="Gemini model id (e.g. gemini-3.7-flash)" - ) - args = parser.parse_args() - globals()["MODEL"] = args.model - - if args.evaluate_only: - run_evaluation() - elif args.predict_only: - asyncio.run(run_predictions()) - else: - asyncio.run(run_predictions()) - run_evaluation() - - -if __name__ == "__main__": - main() diff --git a/benchmarks/ocrbench_v2/ocrbench_v2_gemini_pro_31.py b/benchmarks/ocrbench_v2/ocrbench_v2_gemini_pro_31.py deleted file mode 100644 index e53c652..0000000 --- a/benchmarks/ocrbench_v2/ocrbench_v2_gemini_pro_31.py +++ /dev/null @@ -1,342 +0,0 @@ -""" -OCRBench v2 benchmark for Google Gemini. -10,000 QA pairs across 30 task types (EN + CN). - -Usage: - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_gemini_pro_31 - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_gemini_pro_31 --predict-only - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_gemini_pro_31 --evaluate-only -""" - -import sys -import json -import asyncio -import argparse -from pathlib import Path -from io import BytesIO - -from datasets import load_dataset -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -BENCHMARK_DIR = Path(__file__).resolve().parent -RESULTS_DIR = PROJECT_ROOT / "results" -PRED_OUTPUT = RESULTS_DIR / "ocrbench_v2_gemini_pro_31_predictions.json" -EVAL_OUTPUT = RESULTS_DIR / "ocrbench_v2_gemini_pro_31_scored.json" - -sys.path.insert(0, str(PROJECT_ROOT)) -from src.commons_gemini import invoke_gemini # noqa: E402 - -MODEL = "gemini-3.1-pro-preview" -RATE_LIMIT = 50 -MAX_RETRIES = 3 - -TEXT_SPOTTING_PROMPT_TEMPLATE = """Use OCR on this image to spot all text at {level}. The OCR tool returns each detected text region with its text content and four corner coordinates: top_left, top_right, bottom_left, bottom_right (each as an x,y pixel pair). - -Then use run code to write a Python script that takes those OCR results and: -1. For each text region, compute the axis-aligned bounding box from the four corners: - - x1 = min of all x coordinates (leftmost) - - y1 = min of all y coordinates (topmost) - - x2 = max of all x coordinates (rightmost) - - y2 = max of all y coordinates (bottommost) -2. Normalize each coordinate to the range 0-1000 by dividing by the image width (for x) or height (for y) and multiplying by 1000, then rounding to an integer. -3. Print the results as a Python list. - -Your final answer must be ONLY a Python list in this exact format, with no markdown, no code fences, no explanation: -[(x1, y1, x2, y2, "text"), (x1, y1, x2, y2, "text"), ...]""" - - -def get_spotting_prompt(original_question: str) -> str: - if "line-level" in original_question: - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="line-level") - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="word-level") - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def pil_to_jpeg_bytes(image) -> bytes: - buffer = BytesIO() - image.save(buffer, format="JPEG", quality=95) - return buffer.getvalue() - - -def build_contents(question: str, image_bytes: bytes) -> list: - from google.genai import types - - return [ - types.Part.from_bytes(data=image_bytes, mime_type="image/jpeg"), - question, - ] - - -def extract_text(response) -> str: - text = getattr(response, "text", None) - return text or "" - - -async def process_sample(sample_meta: dict, rate_limiter): - contents = build_contents(sample_meta["question"], sample_meta["image_bytes"]) - - for attempt in range(MAX_RETRIES): - await rate_limiter.acquire() - try: - response = await asyncio.to_thread(invoke_gemini, contents, MODEL) - return extract_text(response) - except Exception as e: - if attempt < MAX_RETRIES - 1: - await asyncio.sleep(2**attempt) - else: - print( - f"Failed after {MAX_RETRIES} attempts for id={sample_meta['id']}: {e}" - ) - return "" - - -BATCH_SIZE = 200 - - -async def run_predictions(): - print("Loading OCRBench v2 from HuggingFace...") - dataset = load_dataset("lmms-lab/OCRBench-v2", split="test") - total = len(dataset) - print(f"Loaded {total} samples") - - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - - existing = {} - if PRED_OUTPUT.exists(): - with open(PRED_OUTPUT) as f: - for item in json.load(f): - if item.get("predict", "") != "": - existing[item["id"]] = item - print(f"Resuming: {len(existing)} successful predictions found, skipping them") - - rate_limiter = RateLimiter(RATE_LIMIT) - output_data = {} - output_data.update(existing) - num_retried = 0 - - from tqdm import tqdm - - for batch_start in tqdm(range(0, total, BATCH_SIZE), desc="Batches"): - batch_end = min(batch_start + BATCH_SIZE, total) - batch = dataset[batch_start:batch_end] - - samples = [] - for i in range(len(batch["id"])): - sample_id = batch["id"][i] - if sample_id in existing: - continue - image_bytes = pil_to_jpeg_bytes(batch["image"][i]) - question = batch["question"][i] - if batch["type"][i] == "text spotting en": - question = get_spotting_prompt(question) - samples.append( - { - "id": sample_id, - "dataset_name": batch["dataset_name"][i], - "type": batch["type"][i], - "question": question, - "answers": batch["answers"][i], - "image_bytes": image_bytes, - } - ) - del batch - - if not samples: - continue - - num_retried += len(samples) - tasks = [process_sample(s, rate_limiter) for s in samples] - predictions = await tqdm_asyncio.gather( - *tasks, desc=f"Predicting {batch_start}-{batch_end}", leave=False - ) - - for sample, pred in zip(samples, predictions): - output_data[sample["id"]] = { - "id": sample["id"], - "dataset_name": sample["dataset_name"], - "type": sample["type"], - "question": sample["question"], - "answers": sample["answers"], - "predict": pred, - } - - del samples, predictions - - final_data = [output_data[i] for i in sorted(output_data.keys())] - with open(PRED_OUTPUT, "w", encoding="utf-8") as f: - json.dump(final_data, f, ensure_ascii=False, indent=2) - - num_failures = sum(1 for d in final_data if d.get("predict", "") == "") - print(f"\nPredictions saved to {PRED_OUTPUT}") - print( - f"Total: {total} | Retried: {num_retried} | Remaining failures: {num_failures}" - ) - - -def run_evaluation(): - if not PRED_OUTPUT.exists(): - print(f"No predictions found at {PRED_OUTPUT}") - print("Run with --predict-only first, or without flags to do both.") - sys.exit(1) - - eval_scripts_dir = BENCHMARK_DIR / "eval_scripts" - sys.path.insert(0, str(eval_scripts_dir)) - - import os - original_cwd = os.getcwd() - os.chdir(BENCHMARK_DIR) - - print("Step 1: Scoring individual samples...") - from benchmarks.ocrbench_v2.eval_scripts.eval import process_predictions # noqa: E402 - - EVAL_OUTPUT.parent.mkdir(parents=True, exist_ok=True) - process_predictions(str(PRED_OUTPUT), str(EVAL_OUTPUT)) - - os.chdir(original_cwd) - print(f"Scored results saved to {EVAL_OUTPUT}") - - print("\nStep 2: Computing overall metrics...") - with open(EVAL_OUTPUT) as f: - scored_data = json.load(f) - - en_scores = { - "text_recognition": [], - "text_detection": [], - "text_spotting": [], - "relationship_extraction": [], - "element_parsing": [], - "mathematical_calculation": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - cn_scores = { - "text_recognition": [], - "relationship_extraction": [], - "element_parsing": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - - type_to_en = { - "text recognition en": "text_recognition", - "fine-grained text recognition en": "text_recognition", - "full-page OCR en": "text_recognition", - "text grounding en": "text_detection", - "VQA with position en": "text_detection", - "text spotting en": "text_spotting", - "key information extraction en": "relationship_extraction", - "key information mapping en": "relationship_extraction", - "document parsing en": "element_parsing", - "chart parsing en": "element_parsing", - "table parsing en": "element_parsing", - "formula recognition en": "element_parsing", - "math QA en": "mathematical_calculation", - "text counting en": "mathematical_calculation", - "document classification en": "visual_text_understanding", - "cognition VQA en": "visual_text_understanding", - "diagram QA en": "visual_text_understanding", - "reasoning VQA en": "knowledge_reasoning", - "science QA en": "knowledge_reasoning", - "APP agent en": "knowledge_reasoning", - "ASCII art classification en": "knowledge_reasoning", - } - type_to_cn = { - "full-page OCR cn": "text_recognition", - "key information extraction cn": "relationship_extraction", - "handwritten answer extraction cn": "relationship_extraction", - "document parsing cn": "element_parsing", - "table parsing cn": "element_parsing", - "formula recognition cn": "element_parsing", - "cognition VQA cn": "visual_text_understanding", - "reasoning VQA cn": "knowledge_reasoning", - "text translation cn": "knowledge_reasoning", - } - - for item in scored_data: - if "ignore" in item: - continue - t = item["type"] - if t in type_to_en: - en_scores[type_to_en[t]].append(item["score"]) - elif t in type_to_cn: - cn_scores[type_to_cn[t]].append(item["score"]) - - def avg(lst): - return sum(lst) / len(lst) if lst else 0.0 - - en_avgs = {k: avg(v) for k, v in en_scores.items() if v} - cn_avgs = {k: avg(v) for k, v in cn_scores.items() if v} - en_overall = avg(list(en_avgs.values())) - cn_overall = avg(list(cn_avgs.values())) - - print(f"\n{'=' * 60}") - print(f"OCRBench v2 Results ({MODEL})") - print(f"{'=' * 60}") - print(f"\n{'Category':<30} {'EN':>8} {'CN':>8}") - print("-" * 48) - all_cats = sorted(set(list(en_scores.keys()) + list(cn_scores.keys()))) - for cat in all_cats: - en_val = f"{en_avgs[cat]:.3f}" if cat in en_avgs else " -" - cn_val = f"{cn_avgs[cat]:.3f}" if cat in cn_avgs else " -" - print(f"{cat:<30} {en_val:>8} {cn_val:>8}") - print("-" * 48) - print(f"{'OVERALL':<30} {en_overall:>8.3f} {cn_overall:>8.3f}") - - metrics = { - "en_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in en_scores.items() - }, - "cn_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in cn_scores.items() - }, - "en_overall": en_overall, - "cn_overall": cn_overall, - "model": MODEL, - } - metrics_path = RESULTS_DIR / "ocrbench_v2_gemini_pro_31_metrics.json" - with open(metrics_path, "w") as f: - json.dump(metrics, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - parser = argparse.ArgumentParser(description="OCRBench v2 benchmark for Google Gemini") - parser.add_argument( - "--predict-only", action="store_true", help="Only generate predictions" - ) - parser.add_argument( - "--evaluate-only", action="store_true", help="Only run evaluation" - ) - args = parser.parse_args() - - if args.evaluate_only: - run_evaluation() - elif args.predict_only: - asyncio.run(run_predictions()) - else: - asyncio.run(run_predictions()) - run_evaluation() - - -if __name__ == "__main__": - main() diff --git a/benchmarks/ocrbench_v2/ocrbench_v2_grok.py b/benchmarks/ocrbench_v2/ocrbench_v2_grok.py deleted file mode 100644 index bd9cc25..0000000 --- a/benchmarks/ocrbench_v2/ocrbench_v2_grok.py +++ /dev/null @@ -1,347 +0,0 @@ -""" -OCRBench v2 benchmark for Grok 4.3 via OpenRouter. -10,000 QA pairs across 30 task types (EN + CN). - -Usage: - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_grok - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_grok --predict-only - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_grok --evaluate-only -""" - -import sys -import json -import asyncio -import argparse -import base64 -from pathlib import Path -from io import BytesIO - -from datasets import load_dataset -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -BENCHMARK_DIR = Path(__file__).resolve().parent -RESULTS_DIR = PROJECT_ROOT / "results" -PRED_OUTPUT = RESULTS_DIR / "ocrbench_v2_grok_predictions.json" -EVAL_OUTPUT = RESULTS_DIR / "ocrbench_v2_grok_scored.json" - -sys.path.insert(0, str(PROJECT_ROOT)) -from src.commons_openrouter import openrouter_client # noqa: E402 - -MODEL = "x-ai/grok-4.3" -RATE_LIMIT = 50 -MAX_RETRIES = 3 - -TEXT_SPOTTING_PROMPT_TEMPLATE = """Use OCR on this image to spot all text at {level}. The OCR tool returns each detected text region with its text content and four corner coordinates: top_left, top_right, bottom_left, bottom_right (each as an x,y pixel pair). - -Then use run code to write a Python script that takes those OCR results and: -1. For each text region, compute the axis-aligned bounding box from the four corners: - - x1 = min of all x coordinates (leftmost) - - y1 = min of all y coordinates (topmost) - - x2 = max of all x coordinates (rightmost) - - y2 = max of all y coordinates (bottommost) -2. Normalize each coordinate to the range 0-1000 by dividing by the image width (for x) or height (for y) and multiplying by 1000, then rounding to an integer. -3. Print the results as a Python list. - -Your final answer must be ONLY a Python list in this exact format, with no markdown, no code fences, no explanation: -[(x1, y1, x2, y2, "text"), (x1, y1, x2, y2, "text"), ...]""" - - -def get_spotting_prompt(original_question: str) -> str: - if "line-level" in original_question: - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="line-level") - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="word-level") - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def pil_to_data_url(image) -> str: - buffer = BytesIO() - image.save(buffer, format="JPEG", quality=95) - b64 = base64.b64encode(buffer.getvalue()).decode("utf-8") - return f"data:image/jpeg;base64,{b64}" - - -def build_messages(question: str, image_url: str) -> list[dict]: - return [ - { - "role": "user", - "content": [ - {"type": "text", "text": question}, - {"type": "image_url", "image_url": {"url": image_url}}, - ], - } - ] - - -async def process_sample(sample_meta: dict, rate_limiter): - messages = build_messages(sample_meta["question"], sample_meta["image_url"]) - - for attempt in range(MAX_RETRIES): - await rate_limiter.acquire() - try: - response = await asyncio.to_thread( - openrouter_client.chat.completions.create, - model=MODEL, - messages=messages, - reasoning_effort="low", - ) - return response.choices[0].message.content or "" - except Exception as e: - if attempt < MAX_RETRIES - 1: - await asyncio.sleep(2**attempt) - else: - print( - f"Failed after {MAX_RETRIES} attempts for id={sample_meta['id']}: {e}" - ) - return "" - - -BATCH_SIZE = 200 - - -async def run_predictions(): - print("Loading OCRBench v2 from HuggingFace...") - dataset = load_dataset("lmms-lab/OCRBench-v2", split="test") - total = len(dataset) - print(f"Loaded {total} samples") - - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - - existing = {} - if PRED_OUTPUT.exists(): - with open(PRED_OUTPUT) as f: - for item in json.load(f): - if item.get("predict", "") != "": - existing[item["id"]] = item - print(f"Resuming: {len(existing)} successful predictions found, skipping them") - - rate_limiter = RateLimiter(RATE_LIMIT) - output_data = {} - output_data.update(existing) - num_retried = 0 - - from tqdm import tqdm - - for batch_start in tqdm(range(0, total, BATCH_SIZE), desc="Batches"): - batch_end = min(batch_start + BATCH_SIZE, total) - batch = dataset[batch_start:batch_end] - - samples = [] - for i in range(len(batch["id"])): - sample_id = batch["id"][i] - if sample_id in existing: - continue - image_url = pil_to_data_url(batch["image"][i]) - question = batch["question"][i] - if batch["type"][i] == "text spotting en": - question = get_spotting_prompt(question) - samples.append( - { - "id": sample_id, - "dataset_name": batch["dataset_name"][i], - "type": batch["type"][i], - "question": question, - "answers": batch["answers"][i], - "image_url": image_url, - } - ) - del batch - - if not samples: - continue - - num_retried += len(samples) - tasks = [process_sample(s, rate_limiter) for s in samples] - predictions = await tqdm_asyncio.gather( - *tasks, desc=f"Predicting {batch_start}-{batch_end}", leave=False - ) - - for sample, pred in zip(samples, predictions): - output_data[sample["id"]] = { - "id": sample["id"], - "dataset_name": sample["dataset_name"], - "type": sample["type"], - "question": sample["question"], - "answers": sample["answers"], - "predict": pred, - } - - del samples, predictions - - final_data = [output_data[i] for i in sorted(output_data.keys())] - with open(PRED_OUTPUT, "w", encoding="utf-8") as f: - json.dump(final_data, f, ensure_ascii=False, indent=2) - - num_failures = sum(1 for d in final_data if d.get("predict", "") == "") - print(f"\nPredictions saved to {PRED_OUTPUT}") - print( - f"Total: {total} | Retried: {num_retried} | Remaining failures: {num_failures}" - ) - - -def run_evaluation(): - if not PRED_OUTPUT.exists(): - print(f"No predictions found at {PRED_OUTPUT}") - print("Run with --predict-only first, or without flags to do both.") - sys.exit(1) - - eval_scripts_dir = BENCHMARK_DIR / "eval_scripts" - sys.path.insert(0, str(eval_scripts_dir)) - - import os - original_cwd = os.getcwd() - os.chdir(BENCHMARK_DIR) - - print("Step 1: Scoring individual samples...") - from benchmarks.ocrbench_v2.eval_scripts.eval import process_predictions # noqa: E402 - - EVAL_OUTPUT.parent.mkdir(parents=True, exist_ok=True) - process_predictions(str(PRED_OUTPUT), str(EVAL_OUTPUT)) - - os.chdir(original_cwd) - print(f"Scored results saved to {EVAL_OUTPUT}") - - print("\nStep 2: Computing overall metrics...") - with open(EVAL_OUTPUT) as f: - scored_data = json.load(f) - - en_scores = { - "text_recognition": [], - "text_detection": [], - "text_spotting": [], - "relationship_extraction": [], - "element_parsing": [], - "mathematical_calculation": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - cn_scores = { - "text_recognition": [], - "relationship_extraction": [], - "element_parsing": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - - type_to_en = { - "text recognition en": "text_recognition", - "fine-grained text recognition en": "text_recognition", - "full-page OCR en": "text_recognition", - "text grounding en": "text_detection", - "VQA with position en": "text_detection", - "text spotting en": "text_spotting", - "key information extraction en": "relationship_extraction", - "key information mapping en": "relationship_extraction", - "document parsing en": "element_parsing", - "chart parsing en": "element_parsing", - "table parsing en": "element_parsing", - "formula recognition en": "element_parsing", - "math QA en": "mathematical_calculation", - "text counting en": "mathematical_calculation", - "document classification en": "visual_text_understanding", - "cognition VQA en": "visual_text_understanding", - "diagram QA en": "visual_text_understanding", - "reasoning VQA en": "knowledge_reasoning", - "science QA en": "knowledge_reasoning", - "APP agent en": "knowledge_reasoning", - "ASCII art classification en": "knowledge_reasoning", - } - type_to_cn = { - "full-page OCR cn": "text_recognition", - "key information extraction cn": "relationship_extraction", - "handwritten answer extraction cn": "relationship_extraction", - "document parsing cn": "element_parsing", - "table parsing cn": "element_parsing", - "formula recognition cn": "element_parsing", - "cognition VQA cn": "visual_text_understanding", - "reasoning VQA cn": "knowledge_reasoning", - "text translation cn": "knowledge_reasoning", - } - - for item in scored_data: - if "ignore" in item: - continue - t = item["type"] - if t in type_to_en: - en_scores[type_to_en[t]].append(item["score"]) - elif t in type_to_cn: - cn_scores[type_to_cn[t]].append(item["score"]) - - def avg(lst): - return sum(lst) / len(lst) if lst else 0.0 - - en_avgs = {k: avg(v) for k, v in en_scores.items() if v} - cn_avgs = {k: avg(v) for k, v in cn_scores.items() if v} - en_overall = avg(list(en_avgs.values())) - cn_overall = avg(list(cn_avgs.values())) - - print(f"\n{'=' * 60}") - print(f"OCRBench v2 Results ({MODEL})") - print(f"{'=' * 60}") - print(f"\n{'Category':<30} {'EN':>8} {'CN':>8}") - print("-" * 48) - all_cats = sorted(set(list(en_scores.keys()) + list(cn_scores.keys()))) - for cat in all_cats: - en_val = f"{en_avgs[cat]:.3f}" if cat in en_avgs else " -" - cn_val = f"{cn_avgs[cat]:.3f}" if cat in cn_avgs else " -" - print(f"{cat:<30} {en_val:>8} {cn_val:>8}") - print("-" * 48) - print(f"{'OVERALL':<30} {en_overall:>8.3f} {cn_overall:>8.3f}") - - metrics = { - "en_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in en_scores.items() - }, - "cn_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in cn_scores.items() - }, - "en_overall": en_overall, - "cn_overall": cn_overall, - "model": MODEL, - } - metrics_path = RESULTS_DIR / "ocrbench_v2_grok_metrics.json" - with open(metrics_path, "w") as f: - json.dump(metrics, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - parser = argparse.ArgumentParser(description="OCRBench v2 benchmark for Grok 4.3 via OpenRouter") - parser.add_argument( - "--predict-only", action="store_true", help="Only generate predictions" - ) - parser.add_argument( - "--evaluate-only", action="store_true", help="Only run evaluation" - ) - args = parser.parse_args() - - if args.evaluate_only: - run_evaluation() - elif args.predict_only: - asyncio.run(run_predictions()) - else: - asyncio.run(run_predictions()) - run_evaluation() - - -if __name__ == "__main__": - main() diff --git a/benchmarks/ocrbench_v2/ocrbench_v2_kimi.py b/benchmarks/ocrbench_v2/ocrbench_v2_kimi.py deleted file mode 100644 index 49a1f6e..0000000 --- a/benchmarks/ocrbench_v2/ocrbench_v2_kimi.py +++ /dev/null @@ -1,346 +0,0 @@ -""" -OCRBench v2 benchmark for Kimi K2.6 via OpenRouter. -10,000 QA pairs across 30 task types (EN + CN). - -Usage: - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_kimi - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_kimi --predict-only - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_kimi --evaluate-only -""" - -import sys -import json -import asyncio -import argparse -import base64 -from pathlib import Path -from io import BytesIO - -from datasets import load_dataset -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -BENCHMARK_DIR = Path(__file__).resolve().parent -RESULTS_DIR = PROJECT_ROOT / "results" -PRED_OUTPUT = RESULTS_DIR / "ocrbench_v2_kimi_predictions.json" -EVAL_OUTPUT = RESULTS_DIR / "ocrbench_v2_kimi_scored.json" - -sys.path.insert(0, str(PROJECT_ROOT)) -from src.commons_openrouter import openrouter_client # noqa: E402 - -MODEL = "moonshotai/kimi-k2.6" -RATE_LIMIT = 50 -MAX_RETRIES = 3 - -TEXT_SPOTTING_PROMPT_TEMPLATE = """Use OCR on this image to spot all text at {level}. The OCR tool returns each detected text region with its text content and four corner coordinates: top_left, top_right, bottom_left, bottom_right (each as an x,y pixel pair). - -Then use run code to write a Python script that takes those OCR results and: -1. For each text region, compute the axis-aligned bounding box from the four corners: - - x1 = min of all x coordinates (leftmost) - - y1 = min of all y coordinates (topmost) - - x2 = max of all x coordinates (rightmost) - - y2 = max of all y coordinates (bottommost) -2. Normalize each coordinate to the range 0-1000 by dividing by the image width (for x) or height (for y) and multiplying by 1000, then rounding to an integer. -3. Print the results as a Python list. - -Your final answer must be ONLY a Python list in this exact format, with no markdown, no code fences, no explanation: -[(x1, y1, x2, y2, "text"), (x1, y1, x2, y2, "text"), ...]""" - - -def get_spotting_prompt(original_question: str) -> str: - if "line-level" in original_question: - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="line-level") - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="word-level") - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def pil_to_data_url(image) -> str: - buffer = BytesIO() - image.save(buffer, format="JPEG", quality=95) - b64 = base64.b64encode(buffer.getvalue()).decode("utf-8") - return f"data:image/jpeg;base64,{b64}" - - -def build_messages(question: str, image_url: str) -> list[dict]: - return [ - { - "role": "user", - "content": [ - {"type": "text", "text": question}, - {"type": "image_url", "image_url": {"url": image_url}}, - ], - } - ] - - -async def process_sample(sample_meta: dict, rate_limiter): - messages = build_messages(sample_meta["question"], sample_meta["image_url"]) - - for attempt in range(MAX_RETRIES): - await rate_limiter.acquire() - try: - response = await asyncio.to_thread( - openrouter_client.chat.completions.create, - model=MODEL, - messages=messages, - ) - return response.choices[0].message.content or "" - except Exception as e: - if attempt < MAX_RETRIES - 1: - await asyncio.sleep(2**attempt) - else: - print( - f"Failed after {MAX_RETRIES} attempts for id={sample_meta['id']}: {e}" - ) - return "" - - -BATCH_SIZE = 200 - - -async def run_predictions(): - print("Loading OCRBench v2 from HuggingFace...") - dataset = load_dataset("lmms-lab/OCRBench-v2", split="test") - total = len(dataset) - print(f"Loaded {total} samples") - - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - - existing = {} - if PRED_OUTPUT.exists(): - with open(PRED_OUTPUT) as f: - for item in json.load(f): - if item.get("predict", "") != "": - existing[item["id"]] = item - print(f"Resuming: {len(existing)} successful predictions found, skipping them") - - rate_limiter = RateLimiter(RATE_LIMIT) - output_data = {} - output_data.update(existing) - num_retried = 0 - - from tqdm import tqdm - - for batch_start in tqdm(range(0, total, BATCH_SIZE), desc="Batches"): - batch_end = min(batch_start + BATCH_SIZE, total) - batch = dataset[batch_start:batch_end] - - samples = [] - for i in range(len(batch["id"])): - sample_id = batch["id"][i] - if sample_id in existing: - continue - image_url = pil_to_data_url(batch["image"][i]) - question = batch["question"][i] - if batch["type"][i] == "text spotting en": - question = get_spotting_prompt(question) - samples.append( - { - "id": sample_id, - "dataset_name": batch["dataset_name"][i], - "type": batch["type"][i], - "question": question, - "answers": batch["answers"][i], - "image_url": image_url, - } - ) - del batch - - if not samples: - continue - - num_retried += len(samples) - tasks = [process_sample(s, rate_limiter) for s in samples] - predictions = await tqdm_asyncio.gather( - *tasks, desc=f"Predicting {batch_start}-{batch_end}", leave=False - ) - - for sample, pred in zip(samples, predictions): - output_data[sample["id"]] = { - "id": sample["id"], - "dataset_name": sample["dataset_name"], - "type": sample["type"], - "question": sample["question"], - "answers": sample["answers"], - "predict": pred, - } - - del samples, predictions - - final_data = [output_data[i] for i in sorted(output_data.keys())] - with open(PRED_OUTPUT, "w", encoding="utf-8") as f: - json.dump(final_data, f, ensure_ascii=False, indent=2) - - num_failures = sum(1 for d in final_data if d.get("predict", "") == "") - print(f"\nPredictions saved to {PRED_OUTPUT}") - print( - f"Total: {total} | Retried: {num_retried} | Remaining failures: {num_failures}" - ) - - -def run_evaluation(): - if not PRED_OUTPUT.exists(): - print(f"No predictions found at {PRED_OUTPUT}") - print("Run with --predict-only first, or without flags to do both.") - sys.exit(1) - - eval_scripts_dir = BENCHMARK_DIR / "eval_scripts" - sys.path.insert(0, str(eval_scripts_dir)) - - import os - original_cwd = os.getcwd() - os.chdir(BENCHMARK_DIR) - - print("Step 1: Scoring individual samples...") - from benchmarks.ocrbench_v2.eval_scripts.eval import process_predictions # noqa: E402 - - EVAL_OUTPUT.parent.mkdir(parents=True, exist_ok=True) - process_predictions(str(PRED_OUTPUT), str(EVAL_OUTPUT)) - - os.chdir(original_cwd) - print(f"Scored results saved to {EVAL_OUTPUT}") - - print("\nStep 2: Computing overall metrics...") - with open(EVAL_OUTPUT) as f: - scored_data = json.load(f) - - en_scores = { - "text_recognition": [], - "text_detection": [], - "text_spotting": [], - "relationship_extraction": [], - "element_parsing": [], - "mathematical_calculation": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - cn_scores = { - "text_recognition": [], - "relationship_extraction": [], - "element_parsing": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - - type_to_en = { - "text recognition en": "text_recognition", - "fine-grained text recognition en": "text_recognition", - "full-page OCR en": "text_recognition", - "text grounding en": "text_detection", - "VQA with position en": "text_detection", - "text spotting en": "text_spotting", - "key information extraction en": "relationship_extraction", - "key information mapping en": "relationship_extraction", - "document parsing en": "element_parsing", - "chart parsing en": "element_parsing", - "table parsing en": "element_parsing", - "formula recognition en": "element_parsing", - "math QA en": "mathematical_calculation", - "text counting en": "mathematical_calculation", - "document classification en": "visual_text_understanding", - "cognition VQA en": "visual_text_understanding", - "diagram QA en": "visual_text_understanding", - "reasoning VQA en": "knowledge_reasoning", - "science QA en": "knowledge_reasoning", - "APP agent en": "knowledge_reasoning", - "ASCII art classification en": "knowledge_reasoning", - } - type_to_cn = { - "full-page OCR cn": "text_recognition", - "key information extraction cn": "relationship_extraction", - "handwritten answer extraction cn": "relationship_extraction", - "document parsing cn": "element_parsing", - "table parsing cn": "element_parsing", - "formula recognition cn": "element_parsing", - "cognition VQA cn": "visual_text_understanding", - "reasoning VQA cn": "knowledge_reasoning", - "text translation cn": "knowledge_reasoning", - } - - for item in scored_data: - if "ignore" in item: - continue - t = item["type"] - if t in type_to_en: - en_scores[type_to_en[t]].append(item["score"]) - elif t in type_to_cn: - cn_scores[type_to_cn[t]].append(item["score"]) - - def avg(lst): - return sum(lst) / len(lst) if lst else 0.0 - - en_avgs = {k: avg(v) for k, v in en_scores.items() if v} - cn_avgs = {k: avg(v) for k, v in cn_scores.items() if v} - en_overall = avg(list(en_avgs.values())) - cn_overall = avg(list(cn_avgs.values())) - - print(f"\n{'=' * 60}") - print(f"OCRBench v2 Results ({MODEL})") - print(f"{'=' * 60}") - print(f"\n{'Category':<30} {'EN':>8} {'CN':>8}") - print("-" * 48) - all_cats = sorted(set(list(en_scores.keys()) + list(cn_scores.keys()))) - for cat in all_cats: - en_val = f"{en_avgs[cat]:.3f}" if cat in en_avgs else " -" - cn_val = f"{cn_avgs[cat]:.3f}" if cat in cn_avgs else " -" - print(f"{cat:<30} {en_val:>8} {cn_val:>8}") - print("-" * 48) - print(f"{'OVERALL':<30} {en_overall:>8.3f} {cn_overall:>8.3f}") - - metrics = { - "en_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in en_scores.items() - }, - "cn_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in cn_scores.items() - }, - "en_overall": en_overall, - "cn_overall": cn_overall, - "model": MODEL, - } - metrics_path = RESULTS_DIR / "ocrbench_v2_kimi_metrics.json" - with open(metrics_path, "w") as f: - json.dump(metrics, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - parser = argparse.ArgumentParser(description="OCRBench v2 benchmark for Kimi K2.6 via OpenRouter") - parser.add_argument( - "--predict-only", action="store_true", help="Only generate predictions" - ) - parser.add_argument( - "--evaluate-only", action="store_true", help="Only run evaluation" - ) - args = parser.parse_args() - - if args.evaluate_only: - run_evaluation() - elif args.predict_only: - asyncio.run(run_predictions()) - else: - asyncio.run(run_predictions()) - run_evaluation() - - -if __name__ == "__main__": - main() diff --git a/benchmarks/ocrbench_v2/ocrbench_v2_openai.py b/benchmarks/ocrbench_v2/ocrbench_v2_openai.py deleted file mode 100644 index d23b756..0000000 --- a/benchmarks/ocrbench_v2/ocrbench_v2_openai.py +++ /dev/null @@ -1,342 +0,0 @@ -""" -OCRBench v2 benchmark for OpenAI. -10,000 QA pairs across 30 task types (EN + CN). - -Usage: - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_openai - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_openai --predict-only - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_openai --evaluate-only -""" - -import sys -import json -import asyncio -import argparse -import base64 -from pathlib import Path -from io import BytesIO - -from datasets import load_dataset -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -BENCHMARK_DIR = Path(__file__).resolve().parent -RESULTS_DIR = PROJECT_ROOT / "results" -PRED_OUTPUT = RESULTS_DIR / "ocrbench_v2_openai_predictions.json" -EVAL_OUTPUT = RESULTS_DIR / "ocrbench_v2_openai_scored.json" - -sys.path.insert(0, str(PROJECT_ROOT)) -from src.commons_openai import invoke_openai # noqa: E402 - -MODEL = "gpt-5.4" -RATE_LIMIT = 25 -MAX_RETRIES = 3 - -TEXT_SPOTTING_PROMPT_TEMPLATE = """Use OCR on this image to spot all text at {level}. The OCR tool returns each detected text region with its text content and four corner coordinates: top_left, top_right, bottom_left, bottom_right (each as an x,y pixel pair). - -Then use run code to write a Python script that takes those OCR results and: -1. For each text region, compute the axis-aligned bounding box from the four corners: - - x1 = min of all x coordinates (leftmost) - - y1 = min of all y coordinates (topmost) - - x2 = max of all x coordinates (rightmost) - - y2 = max of all y coordinates (bottommost) -2. Normalize each coordinate to the range 0-1000 by dividing by the image width (for x) or height (for y) and multiplying by 1000, then rounding to an integer. -3. Print the results as a Python list. - -Your final answer must be ONLY a Python list in this exact format, with no markdown, no code fences, no explanation: -[(x1, y1, x2, y2, "text"), (x1, y1, x2, y2, "text"), ...]""" - - -def get_spotting_prompt(original_question: str) -> str: - if "line-level" in original_question: - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="line-level") - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="word-level") - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def pil_to_data_url(image) -> str: - buffer = BytesIO() - image.save(buffer, format="JPEG", quality=95) - b64 = base64.b64encode(buffer.getvalue()).decode("utf-8") - return f"data:image/jpeg;base64,{b64}" - - -def build_messages(question: str, image_url: str) -> list[dict]: - return [ - { - "role": "user", - "content": [ - {"type": "text", "text": question}, - {"type": "image_url", "image_url": {"url": image_url}}, - ], - } - ] - - -async def process_sample(sample_meta: dict, rate_limiter): - messages = build_messages(sample_meta["question"], sample_meta["image_url"]) - - for attempt in range(MAX_RETRIES): - await rate_limiter.acquire() - try: - response = await asyncio.to_thread(invoke_openai, messages, MODEL) - return response.choices[0].message.content or "" - except Exception as e: - if attempt < MAX_RETRIES - 1: - await asyncio.sleep(2**attempt) - else: - print( - f"Failed after {MAX_RETRIES} attempts for id={sample_meta['id']}: {e}" - ) - return "" - - -BATCH_SIZE = 100 - - -async def run_predictions(): - print("Loading OCRBench v2 from HuggingFace...") - dataset = load_dataset("lmms-lab/OCRBench-v2", split="test") - total = len(dataset) - print(f"Loaded {total} samples") - - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - - existing = {} - if PRED_OUTPUT.exists(): - with open(PRED_OUTPUT) as f: - for item in json.load(f): - if item.get("predict", "") != "": - existing[item["id"]] = item - print(f"Resuming: {len(existing)} successful predictions found, skipping them") - - rate_limiter = RateLimiter(RATE_LIMIT) - output_data = {} - output_data.update(existing) - num_retried = 0 - - from tqdm import tqdm - - for batch_start in tqdm(range(0, total, BATCH_SIZE), desc="Batches"): - batch_end = min(batch_start + BATCH_SIZE, total) - batch = dataset[batch_start:batch_end] - - samples = [] - for i in range(len(batch["id"])): - sample_id = batch["id"][i] - if sample_id in existing: - continue - image_url = pil_to_data_url(batch["image"][i]) - question = batch["question"][i] - if batch["type"][i] == "text spotting en": - question = get_spotting_prompt(question) - samples.append( - { - "id": sample_id, - "dataset_name": batch["dataset_name"][i], - "type": batch["type"][i], - "question": question, - "answers": batch["answers"][i], - "image_url": image_url, - } - ) - del batch - - if not samples: - continue - - num_retried += len(samples) - tasks = [process_sample(s, rate_limiter) for s in samples] - predictions = await tqdm_asyncio.gather( - *tasks, desc=f"Predicting {batch_start}-{batch_end}", leave=False - ) - - for sample, pred in zip(samples, predictions): - output_data[sample["id"]] = { - "id": sample["id"], - "dataset_name": sample["dataset_name"], - "type": sample["type"], - "question": sample["question"], - "answers": sample["answers"], - "predict": pred, - } - - del samples, predictions - - final_data = [output_data[i] for i in sorted(output_data.keys())] - with open(PRED_OUTPUT, "w", encoding="utf-8") as f: - json.dump(final_data, f, ensure_ascii=False, indent=2) - - num_failures = sum(1 for d in final_data if d.get("predict", "") == "") - print(f"\nPredictions saved to {PRED_OUTPUT}") - print( - f"Total: {total} | Retried: {num_retried} | Remaining failures: {num_failures}" - ) - - -def run_evaluation(): - if not PRED_OUTPUT.exists(): - print(f"No predictions found at {PRED_OUTPUT}") - print("Run with --predict-only first, or without flags to do both.") - sys.exit(1) - - eval_scripts_dir = BENCHMARK_DIR / "eval_scripts" - sys.path.insert(0, str(eval_scripts_dir)) - - import os - original_cwd = os.getcwd() - os.chdir(BENCHMARK_DIR) - - print("Step 1: Scoring individual samples...") - from benchmarks.ocrbench_v2.eval_scripts.eval import process_predictions # noqa: E402 - - EVAL_OUTPUT.parent.mkdir(parents=True, exist_ok=True) - process_predictions(str(PRED_OUTPUT), str(EVAL_OUTPUT)) - - os.chdir(original_cwd) - print(f"Scored results saved to {EVAL_OUTPUT}") - - print("\nStep 2: Computing overall metrics...") - with open(EVAL_OUTPUT) as f: - scored_data = json.load(f) - - en_scores = { - "text_recognition": [], - "text_detection": [], - "text_spotting": [], - "relationship_extraction": [], - "element_parsing": [], - "mathematical_calculation": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - cn_scores = { - "text_recognition": [], - "relationship_extraction": [], - "element_parsing": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - - type_to_en = { - "text recognition en": "text_recognition", - "fine-grained text recognition en": "text_recognition", - "full-page OCR en": "text_recognition", - "text grounding en": "text_detection", - "VQA with position en": "text_detection", - "text spotting en": "text_spotting", - "key information extraction en": "relationship_extraction", - "key information mapping en": "relationship_extraction", - "document parsing en": "element_parsing", - "chart parsing en": "element_parsing", - "table parsing en": "element_parsing", - "formula recognition en": "element_parsing", - "math QA en": "mathematical_calculation", - "text counting en": "mathematical_calculation", - "document classification en": "visual_text_understanding", - "cognition VQA en": "visual_text_understanding", - "diagram QA en": "visual_text_understanding", - "reasoning VQA en": "knowledge_reasoning", - "science QA en": "knowledge_reasoning", - "APP agent en": "knowledge_reasoning", - "ASCII art classification en": "knowledge_reasoning", - } - type_to_cn = { - "full-page OCR cn": "text_recognition", - "key information extraction cn": "relationship_extraction", - "handwritten answer extraction cn": "relationship_extraction", - "document parsing cn": "element_parsing", - "table parsing cn": "element_parsing", - "formula recognition cn": "element_parsing", - "cognition VQA cn": "visual_text_understanding", - "reasoning VQA cn": "knowledge_reasoning", - "text translation cn": "knowledge_reasoning", - } - - for item in scored_data: - if "ignore" in item: - continue - t = item["type"] - if t in type_to_en: - en_scores[type_to_en[t]].append(item["score"]) - elif t in type_to_cn: - cn_scores[type_to_cn[t]].append(item["score"]) - - def avg(lst): - return sum(lst) / len(lst) if lst else 0.0 - - en_avgs = {k: avg(v) for k, v in en_scores.items() if v} - cn_avgs = {k: avg(v) for k, v in cn_scores.items() if v} - en_overall = avg(list(en_avgs.values())) - cn_overall = avg(list(cn_avgs.values())) - - print(f"\n{'=' * 60}") - print(f"OCRBench v2 Results ({MODEL})") - print(f"{'=' * 60}") - print(f"\n{'Category':<30} {'EN':>8} {'CN':>8}") - print("-" * 48) - all_cats = sorted(set(list(en_scores.keys()) + list(cn_scores.keys()))) - for cat in all_cats: - en_val = f"{en_avgs[cat]:.3f}" if cat in en_avgs else " -" - cn_val = f"{cn_avgs[cat]:.3f}" if cat in cn_avgs else " -" - print(f"{cat:<30} {en_val:>8} {cn_val:>8}") - print("-" * 48) - print(f"{'OVERALL':<30} {en_overall:>8.3f} {cn_overall:>8.3f}") - - metrics = { - "en_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in en_scores.items() - }, - "cn_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in cn_scores.items() - }, - "en_overall": en_overall, - "cn_overall": cn_overall, - "model": MODEL, - } - metrics_path = RESULTS_DIR / "ocrbench_v2_openai_metrics.json" - with open(metrics_path, "w") as f: - json.dump(metrics, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - parser = argparse.ArgumentParser(description="OCRBench v2 benchmark for OpenAI") - parser.add_argument( - "--predict-only", action="store_true", help="Only generate predictions" - ) - parser.add_argument( - "--evaluate-only", action="store_true", help="Only run evaluation" - ) - args = parser.parse_args() - - if args.evaluate_only: - run_evaluation() - elif args.predict_only: - asyncio.run(run_predictions()) - else: - asyncio.run(run_predictions()) - run_evaluation() - - -if __name__ == "__main__": - main() diff --git a/benchmarks/ocrbench_v2/ocrbench_v2_openai_mini.py b/benchmarks/ocrbench_v2/ocrbench_v2_openai_mini.py deleted file mode 100644 index 8c5329b..0000000 --- a/benchmarks/ocrbench_v2/ocrbench_v2_openai_mini.py +++ /dev/null @@ -1,342 +0,0 @@ -""" -OCRBench v2 benchmark for OpenAI gpt-5.4-mini. -10,000 QA pairs across 30 task types (EN + CN). - -Usage: - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_openai_mini - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_openai_mini --predict-only - uv run -m benchmarks.ocrbench_v2.ocrbench_v2_openai_mini --evaluate-only -""" - -import sys -import json -import asyncio -import argparse -import base64 -from pathlib import Path -from io import BytesIO - -from datasets import load_dataset -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -BENCHMARK_DIR = Path(__file__).resolve().parent -RESULTS_DIR = PROJECT_ROOT / "results" -PRED_OUTPUT = RESULTS_DIR / "ocrbench_v2_openai_mini_predictions.json" -EVAL_OUTPUT = RESULTS_DIR / "ocrbench_v2_openai_mini_scored.json" - -sys.path.insert(0, str(PROJECT_ROOT)) -from src.commons_openai import invoke_openai # noqa: E402 - -MODEL = "gpt-5.4-mini" -RATE_LIMIT = 50 -MAX_RETRIES = 3 - -TEXT_SPOTTING_PROMPT_TEMPLATE = """Use OCR on this image to spot all text at {level}. The OCR tool returns each detected text region with its text content and four corner coordinates: top_left, top_right, bottom_left, bottom_right (each as an x,y pixel pair). - -Then use run code to write a Python script that takes those OCR results and: -1. For each text region, compute the axis-aligned bounding box from the four corners: - - x1 = min of all x coordinates (leftmost) - - y1 = min of all y coordinates (topmost) - - x2 = max of all x coordinates (rightmost) - - y2 = max of all y coordinates (bottommost) -2. Normalize each coordinate to the range 0-1000 by dividing by the image width (for x) or height (for y) and multiplying by 1000, then rounding to an integer. -3. Print the results as a Python list. - -Your final answer must be ONLY a Python list in this exact format, with no markdown, no code fences, no explanation: -[(x1, y1, x2, y2, "text"), (x1, y1, x2, y2, "text"), ...]""" - - -def get_spotting_prompt(original_question: str) -> str: - if "line-level" in original_question: - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="line-level") - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="word-level") - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def pil_to_data_url(image) -> str: - buffer = BytesIO() - image.save(buffer, format="JPEG", quality=95) - b64 = base64.b64encode(buffer.getvalue()).decode("utf-8") - return f"data:image/jpeg;base64,{b64}" - - -def build_messages(question: str, image_url: str) -> list[dict]: - return [ - { - "role": "user", - "content": [ - {"type": "text", "text": question}, - {"type": "image_url", "image_url": {"url": image_url}}, - ], - } - ] - - -async def process_sample(sample_meta: dict, rate_limiter): - messages = build_messages(sample_meta["question"], sample_meta["image_url"]) - - for attempt in range(MAX_RETRIES): - await rate_limiter.acquire() - try: - response = await asyncio.to_thread(invoke_openai, messages, MODEL) - return response.choices[0].message.content or "" - except Exception as e: - if attempt < MAX_RETRIES - 1: - await asyncio.sleep(2**attempt) - else: - print( - f"Failed after {MAX_RETRIES} attempts for id={sample_meta['id']}: {e}" - ) - return "" - - -BATCH_SIZE = 200 - - -async def run_predictions(): - print("Loading OCRBench v2 from HuggingFace...") - dataset = load_dataset("lmms-lab/OCRBench-v2", split="test") - total = len(dataset) - print(f"Loaded {total} samples") - - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - - existing = {} - if PRED_OUTPUT.exists(): - with open(PRED_OUTPUT) as f: - for item in json.load(f): - if item.get("predict", "") != "": - existing[item["id"]] = item - print(f"Resuming: {len(existing)} successful predictions found, skipping them") - - rate_limiter = RateLimiter(RATE_LIMIT) - output_data = {} - output_data.update(existing) - num_retried = 0 - - from tqdm import tqdm - - for batch_start in tqdm(range(0, total, BATCH_SIZE), desc="Batches"): - batch_end = min(batch_start + BATCH_SIZE, total) - batch = dataset[batch_start:batch_end] - - samples = [] - for i in range(len(batch["id"])): - sample_id = batch["id"][i] - if sample_id in existing: - continue - image_url = pil_to_data_url(batch["image"][i]) - question = batch["question"][i] - if batch["type"][i] == "text spotting en": - question = get_spotting_prompt(question) - samples.append( - { - "id": sample_id, - "dataset_name": batch["dataset_name"][i], - "type": batch["type"][i], - "question": question, - "answers": batch["answers"][i], - "image_url": image_url, - } - ) - del batch - - if not samples: - continue - - num_retried += len(samples) - tasks = [process_sample(s, rate_limiter) for s in samples] - predictions = await tqdm_asyncio.gather( - *tasks, desc=f"Predicting {batch_start}-{batch_end}", leave=False - ) - - for sample, pred in zip(samples, predictions): - output_data[sample["id"]] = { - "id": sample["id"], - "dataset_name": sample["dataset_name"], - "type": sample["type"], - "question": sample["question"], - "answers": sample["answers"], - "predict": pred, - } - - del samples, predictions - - final_data = [output_data[i] for i in sorted(output_data.keys())] - with open(PRED_OUTPUT, "w", encoding="utf-8") as f: - json.dump(final_data, f, ensure_ascii=False, indent=2) - - num_failures = sum(1 for d in final_data if d.get("predict", "") == "") - print(f"\nPredictions saved to {PRED_OUTPUT}") - print( - f"Total: {total} | Retried: {num_retried} | Remaining failures: {num_failures}" - ) - - -def run_evaluation(): - if not PRED_OUTPUT.exists(): - print(f"No predictions found at {PRED_OUTPUT}") - print("Run with --predict-only first, or without flags to do both.") - sys.exit(1) - - eval_scripts_dir = BENCHMARK_DIR / "eval_scripts" - sys.path.insert(0, str(eval_scripts_dir)) - - import os - original_cwd = os.getcwd() - os.chdir(BENCHMARK_DIR) - - print("Step 1: Scoring individual samples...") - from benchmarks.ocrbench_v2.eval_scripts.eval import process_predictions # noqa: E402 - - EVAL_OUTPUT.parent.mkdir(parents=True, exist_ok=True) - process_predictions(str(PRED_OUTPUT), str(EVAL_OUTPUT)) - - os.chdir(original_cwd) - print(f"Scored results saved to {EVAL_OUTPUT}") - - print("\nStep 2: Computing overall metrics...") - with open(EVAL_OUTPUT) as f: - scored_data = json.load(f) - - en_scores = { - "text_recognition": [], - "text_detection": [], - "text_spotting": [], - "relationship_extraction": [], - "element_parsing": [], - "mathematical_calculation": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - cn_scores = { - "text_recognition": [], - "relationship_extraction": [], - "element_parsing": [], - "visual_text_understanding": [], - "knowledge_reasoning": [], - } - - type_to_en = { - "text recognition en": "text_recognition", - "fine-grained text recognition en": "text_recognition", - "full-page OCR en": "text_recognition", - "text grounding en": "text_detection", - "VQA with position en": "text_detection", - "text spotting en": "text_spotting", - "key information extraction en": "relationship_extraction", - "key information mapping en": "relationship_extraction", - "document parsing en": "element_parsing", - "chart parsing en": "element_parsing", - "table parsing en": "element_parsing", - "formula recognition en": "element_parsing", - "math QA en": "mathematical_calculation", - "text counting en": "mathematical_calculation", - "document classification en": "visual_text_understanding", - "cognition VQA en": "visual_text_understanding", - "diagram QA en": "visual_text_understanding", - "reasoning VQA en": "knowledge_reasoning", - "science QA en": "knowledge_reasoning", - "APP agent en": "knowledge_reasoning", - "ASCII art classification en": "knowledge_reasoning", - } - type_to_cn = { - "full-page OCR cn": "text_recognition", - "key information extraction cn": "relationship_extraction", - "handwritten answer extraction cn": "relationship_extraction", - "document parsing cn": "element_parsing", - "table parsing cn": "element_parsing", - "formula recognition cn": "element_parsing", - "cognition VQA cn": "visual_text_understanding", - "reasoning VQA cn": "knowledge_reasoning", - "text translation cn": "knowledge_reasoning", - } - - for item in scored_data: - if "ignore" in item: - continue - t = item["type"] - if t in type_to_en: - en_scores[type_to_en[t]].append(item["score"]) - elif t in type_to_cn: - cn_scores[type_to_cn[t]].append(item["score"]) - - def avg(lst): - return sum(lst) / len(lst) if lst else 0.0 - - en_avgs = {k: avg(v) for k, v in en_scores.items() if v} - cn_avgs = {k: avg(v) for k, v in cn_scores.items() if v} - en_overall = avg(list(en_avgs.values())) - cn_overall = avg(list(cn_avgs.values())) - - print(f"\n{'=' * 60}") - print(f"OCRBench v2 Results ({MODEL})") - print(f"{'=' * 60}") - print(f"\n{'Category':<30} {'EN':>8} {'CN':>8}") - print("-" * 48) - all_cats = sorted(set(list(en_scores.keys()) + list(cn_scores.keys()))) - for cat in all_cats: - en_val = f"{en_avgs[cat]:.3f}" if cat in en_avgs else " -" - cn_val = f"{cn_avgs[cat]:.3f}" if cat in cn_avgs else " -" - print(f"{cat:<30} {en_val:>8} {cn_val:>8}") - print("-" * 48) - print(f"{'OVERALL':<30} {en_overall:>8.3f} {cn_overall:>8.3f}") - - metrics = { - "en_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in en_scores.items() - }, - "cn_scores": { - k: {"avg": avg(v), "count": len(v)} for k, v in cn_scores.items() - }, - "en_overall": en_overall, - "cn_overall": cn_overall, - "model": MODEL, - } - metrics_path = RESULTS_DIR / "ocrbench_v2_openai_mini_metrics.json" - with open(metrics_path, "w") as f: - json.dump(metrics, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - parser = argparse.ArgumentParser(description="OCRBench v2 benchmark for OpenAI gpt-5.4-mini") - parser.add_argument( - "--predict-only", action="store_true", help="Only generate predictions" - ) - parser.add_argument( - "--evaluate-only", action="store_true", help="Only run evaluation" - ) - args = parser.parse_args() - - if args.evaluate_only: - run_evaluation() - elif args.predict_only: - asyncio.run(run_predictions()) - else: - asyncio.run(run_predictions()) - run_evaluation() - - -if __name__ == "__main__": - main() diff --git a/benchmarks/ocrbench_v2/ocrbench_v2_text_spotting_en.py b/benchmarks/ocrbench_v2/ocrbench_v2_text_spotting_en.py deleted file mode 100644 index b0eeb09..0000000 --- a/benchmarks/ocrbench_v2/ocrbench_v2_text_spotting_en.py +++ /dev/null @@ -1,376 +0,0 @@ -""" -OCRBench v2 benchmark for Interfaze — Text Spotting EN only. -Runs predictions and evaluation exclusively for the "text spotting en" task type. - -Usage: - uv run python benchmarks/ocrbench_v2/ocrbench_v2_text_spotting_en.py - uv run python benchmarks/ocrbench_v2/ocrbench_v2_text_spotting_en.py --predict-only - uv run python benchmarks/ocrbench_v2/ocrbench_v2_text_spotting_en.py --evaluate-only -""" - -import sys -import json -import asyncio -import argparse -import base64 -from pathlib import Path -from io import BytesIO - -from datasets import load_dataset -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -BENCHMARK_DIR = Path(__file__).resolve().parent -RESULTS_DIR = PROJECT_ROOT / "results" -PRED_OUTPUT = RESULTS_DIR / "ocrbench_v2_text_spotting_en_predictions.json" -EVAL_OUTPUT = RESULTS_DIR / "ocrbench_v2_text_spotting_en_scored.json" - -sys.path.insert(0, str(PROJECT_ROOT)) -from src.commons import invoke_interfaze # noqa: E402 - -RATE_LIMIT = 25 -MAX_RETRIES = 3 -TARGET_TYPE = "text spotting en" - -TEXT_SPOTTING_PROMPT_TEMPLATE = """Use OCR on this image to spot all text at {level}. The OCR tool returns each detected text region with its text content and four corner coordinates: top_left, top_right, bottom_left, bottom_right (each as an x,y pixel pair). - -Then use run code to write a Python script that takes those OCR results and: -1. For each text region, compute the axis-aligned bounding box from the four corners: - - x1 = min of all x coordinates (leftmost) - - y1 = min of all y coordinates (topmost) - - x2 = max of all x coordinates (rightmost) - - y2 = max of all y coordinates (bottommost) -2. Normalize each coordinate to the range 0-1000 by dividing by the image width (for x) or height (for y) and multiplying by 1000, then rounding to an integer. -3. Print the results as a Python list. - -Your final answer must be ONLY a Python list in this exact format, with no markdown, no code fences, no explanation: -[(x1, y1, x2, y2, "text"), (x1, y1, x2, y2, "text"), ...]""" - - -def get_spotting_prompt(original_question: str) -> str: - """Preserve word-level vs line-level from the original question.""" - if "line-level" in original_question: - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="line-level") - return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level="word-level") - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def pil_to_data_url(image) -> str: - buffer = BytesIO() - image.save(buffer, format="JPEG", quality=95) - b64 = base64.b64encode(buffer.getvalue()).decode("utf-8") - return f"data:image/jpeg;base64,{b64}" - - -def build_messages(question: str, image_url: str) -> list[dict]: - return [ - { - "role": "user", - "content": [ - {"type": "text", "text": question}, - {"type": "image_url", "image_url": {"url": image_url}}, - ], - } - ] - - -async def process_sample(sample_meta: dict, rate_limiter): - messages = build_messages(sample_meta["question"], sample_meta["image_url"]) - - for attempt in range(MAX_RETRIES): - await rate_limiter.acquire() - try: - response = await asyncio.to_thread(invoke_interfaze, messages) - return response.choices[0].message.content - except Exception as e: - if attempt < MAX_RETRIES - 1: - await asyncio.sleep(2**attempt) - else: - print( - f"Failed after {MAX_RETRIES} attempts for id={sample_meta['id']}: {e}" - ) - return "" - - -BATCH_SIZE = 20 - - -async def run_predictions(): - print("Loading OCRBench v2 from HuggingFace...") - dataset = load_dataset("lmms-lab/OCRBench-v2", split="test") - total = len(dataset) - print(f"Loaded {total} samples") - - # Filter to only "text spotting en" samples - spotting_indices = [i for i in range(total) if dataset[i]["type"] == TARGET_TYPE] - print(f"Filtered to {len(spotting_indices)} '{TARGET_TYPE}' samples") - - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - - # Always run fresh - if PRED_OUTPUT.exists(): - PRED_OUTPUT.unlink() - print("Cleared previous predictions — running fresh") - - rate_limiter = RateLimiter(RATE_LIMIT) - output_data = {} - num_retried = 0 - - # Import eval functions for realtime scoring - eval_scripts_dir = BENCHMARK_DIR / "eval_scripts" - sys.path.insert(0, str(eval_scripts_dir)) - import os - - original_cwd = os.getcwd() - os.chdir(BENCHMARK_DIR) - from eval_text_spotting_en import ( - extract_bounding_boxes, - spotting_evaluation_normalized, - ) - - def parse_gt(answers): - """Parse ground truth polygons into axis-aligned bboxes + text.""" - bboxes, contents = [], [] - for line in answers[0].strip().split("\n"): - parts = line.split(",") - num_coords = 0 - for p in parts: - try: - int(p.strip()) - num_coords += 1 - except ValueError: - break - if num_coords >= 8 and num_coords < len(parts): - coords = [int(p.strip()) for p in parts[:num_coords]] - text = ",".join(parts[num_coords:]) - x_coords = coords[0::2] - y_coords = coords[1::2] - x1, y1 = min(x_coords), min(y_coords) - x2, y2 = max(x_coords), max(y_coords) - bboxes.append([x1, y1, x2, y1, x2, y2, x1, y2]) - contents.append(text) - elif num_coords >= 8 and num_coords == len(parts) and num_coords % 2 == 1: - coords = [int(p.strip()) for p in parts[:num_coords - 1]] - text = parts[num_coords - 1].strip() - x_coords = coords[0::2] - y_coords = coords[1::2] - x1, y1 = min(x_coords), min(y_coords) - x2, y2 = max(x_coords), max(y_coords) - bboxes.append([x1, y1, x2, y1, x2, y2, x1, y2]) - contents.append(text) - return bboxes, contents - - def quick_score(predict_str, answers): - """Score a single prediction inline.""" - try: - bboxes, contents = parse_gt(answers) - if not bboxes: - return 0.0 - pred_boxes = extract_bounding_boxes(predict_str) - if not pred_boxes: - return 0.0 - return spotting_evaluation_normalized( - pred_boxes, {"bbox": bboxes, "content": contents} - ) - except Exception: - return 0.0 - - from tqdm import tqdm - - all_scores = [] - num_batches = (len(spotting_indices) + BATCH_SIZE - 1) // BATCH_SIZE - - for batch_num in range(num_batches): - batch_start = batch_num * BATCH_SIZE - batch_end = min(batch_start + BATCH_SIZE, len(spotting_indices)) - batch_indices = spotting_indices[batch_start:batch_end] - - # Prepare all samples in this batch - samples = [] - for idx in batch_indices: - row = dataset[idx] - image_url = pil_to_data_url(row["image"]) - samples.append({ - "id": row["id"], - "dataset_name": row["dataset_name"], - "type": row["type"], - "question": get_spotting_prompt(row["question"]), - "answers": row["answers"], - "image_url": image_url, - }) - - # Fire all requests in parallel - print(f"Batch {batch_num + 1}/{num_batches} — sending {len(samples)} requests in parallel...") - tasks = [process_sample(s, rate_limiter) for s in samples] - predictions = await tqdm_asyncio.gather( - *tasks, - desc=f"Batch {batch_num + 1}/{num_batches}", - leave=True, - ) - - # Score all results - batch_scores = [] - for sample, pred in zip(samples, predictions): - num_retried += 1 - score = quick_score(pred, sample["answers"]) - batch_scores.append(score) - all_scores.append(score) - - output_data[sample["id"]] = { - "id": sample["id"], - "dataset_name": sample["dataset_name"], - "type": sample["type"], - "question": sample["question"], - "answers": sample["answers"], - "predict": pred, - } - - # Save after each batch - final_data = [output_data[i] for i in sorted(output_data.keys())] - with open(PRED_OUTPUT, "w", encoding="utf-8") as f: - json.dump(final_data, f, ensure_ascii=False, indent=2) - - batch_avg = sum(batch_scores) / len(batch_scores) if batch_scores else 0.0 - overall_avg = sum(all_scores) / len(all_scores) if all_scores else 0.0 - print( - f" Batch {batch_num + 1} done — batch H-mean: {batch_avg:.4f} | overall H-mean: {overall_avg:.4f}" - ) - - os.chdir(original_cwd) - - final_data = [output_data[i] for i in sorted(output_data.keys())] - num_failures = sum(1 for d in final_data if d.get("predict", "") == "") - final_hmean = sum(all_scores) / len(all_scores) if all_scores else 0.0 - print(f"\nPredictions saved to {PRED_OUTPUT}") - print( - f"Total: {len(spotting_indices)} | Retried: {num_retried} | Remaining failures: {num_failures}" - ) - print(f"Final H-mean: {final_hmean:.4f}" - ) - - -def run_evaluation(): - if not PRED_OUTPUT.exists(): - print(f"No predictions found at {PRED_OUTPUT}") - print("Run with --predict-only first, or without flags to do both.") - sys.exit(1) - - eval_scripts_dir = BENCHMARK_DIR / "eval_scripts" - sys.path.insert(0, str(eval_scripts_dir)) - - # spotting_metric.py uses relative paths like ./eval_scripts/spotting_eval/submit - # so we must chdir to the benchmark dir for the RRC evaluation to work - import os - - original_cwd = os.getcwd() - os.chdir(BENCHMARK_DIR) - - # Step 1: Score each sample using eval_text_spotting_en.py - print("Step 1: Scoring text spotting EN samples...") - from eval_text_spotting_en import process_predictions # noqa: E402 - - EVAL_OUTPUT.parent.mkdir(parents=True, exist_ok=True) - process_predictions(str(PRED_OUTPUT), str(EVAL_OUTPUT)) - - os.chdir(original_cwd) - print(f"Scored results saved to {EVAL_OUTPUT}") - - # Step 2: Compute text spotting metrics - print("\nStep 2: Computing text spotting EN metrics...") - with open(EVAL_OUTPUT) as f: - scored_data = json.load(f) - - scores = [] - for item in scored_data: - if "ignore" in item: - continue - if item["type"] == TARGET_TYPE: - scores.append(item["score"]) - - avg_score = sum(scores) / len(scores) if scores else 0.0 - - print(f"\n{'=' * 60}") - print("OCRBench v2 — Text Spotting EN Results (Interfaze)") - print(f"{'=' * 60}") - print(f" Samples: {len(scores)}") - print(f" H-mean: {avg_score:.4f}") - print(f"{'=' * 60}") - - # Save metrics - metrics = { - "text_spotting_en": { - "avg": avg_score, - "count": len(scores), - }, - "model": "interfaze-beta", - } - metrics_path = RESULTS_DIR / "ocrbench_v2_text_spotting_en_metrics.json" - with open(metrics_path, "w") as f: - json.dump(metrics, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - # Also update the main metrics file's text_spotting entry - main_metrics_path = RESULTS_DIR / "ocrbench_v2_metrics.json" - if main_metrics_path.exists(): - with open(main_metrics_path) as f: - main_metrics = json.load(f) - main_metrics["en_scores"]["text_spotting"] = { - "avg": avg_score, - "count": len(scores), - } - # Recompute en_overall - en_avgs = { - k: v["avg"] - for k, v in main_metrics["en_scores"].items() - if v.get("count", 0) > 0 - } - main_metrics["en_overall"] = ( - sum(en_avgs.values()) / len(en_avgs) if en_avgs else 0.0 - ) - with open(main_metrics_path, "w") as f: - json.dump(main_metrics, f, indent=2) - print(f"Updated text_spotting in {main_metrics_path}") - - -def main(): - parser = argparse.ArgumentParser( - description="OCRBench v2 — Text Spotting EN benchmark for Interfaze" - ) - parser.add_argument( - "--predict-only", action="store_true", help="Only generate predictions" - ) - parser.add_argument( - "--evaluate-only", action="store_true", help="Only run evaluation" - ) - args = parser.parse_args() - - if args.evaluate_only: - run_evaluation() - elif args.predict_only: - asyncio.run(run_predictions()) - else: - asyncio.run(run_predictions()) - run_evaluation() - - -if __name__ == "__main__": - main() diff --git a/scripts/run_fireworks_suite.sh b/scripts/run_fireworks_suite.sh index 6dabcf3..0884ba1 100755 --- a/scripts/run_fireworks_suite.sh +++ b/scripts/run_fireworks_suite.sh @@ -102,8 +102,11 @@ else echo " [skip] mmmlu: set TARGET= (python -m bench_core list-targets)" fi -run_step ocrbench uv run -m benchmarks.ocrbench_v2.ocrbench_v2_fireworks --model "$MODEL" \ - --provider "$PROVIDER" --reasoning "$BENCH_OPENROUTER_EFFORT" +if [ -n "${TARGET:-}" ]; then + run_step ocrbench uv run python -m bench_core run --benchmark ocrbench_v2 --target "$TARGET" +else + echo " [skip] ocrbench: set TARGET= (python -m bench_core list-targets)" +fi # Gated dataset — needs `hf auth login` plus accepted terms on the Hub. # GPQA now runs through the unified CLI; provider/model/reasoning come from the diff --git a/tests/test_ocrbench_bench.py b/tests/test_ocrbench_bench.py new file mode 100644 index 0000000..ba3e327 --- /dev/null +++ b/tests/test_ocrbench_bench.py @@ -0,0 +1,53 @@ +"""OCRBench v2: offline parity of the category aggregation. + +The per-sample scorer is reused verbatim (not re-tested here). This replays each +archived scored.json — which already carries per-sample `score` + `type` — through +the ported `aggregate` and matches en_overall/cn_overall + per-category means. +""" + +import json +import pathlib + +import pytest + +from benchmarks.ocrbench_v2 import bench + +RESULTS = pathlib.Path(__file__).resolve().parent.parent / "results" +ARCHIVED = [ + "ocrbench_v2_fireworks_inkling", + "ocrbench_v2_fireworks_inkling-small", + "ocrbench_v2_gemini", +] + + +def test_aggregate_macro_of_means(): + scored = [ + {"type": "text recognition en", "score": 1.0}, + {"type": "text recognition en", "score": 0.0}, # text_recognition avg = 0.5 + {"type": "math QA en", "score": 1.0}, # mathematical_calculation avg = 1.0 + {"type": "text spotting en", "score": 0.0, "ignore": "True"}, # skipped + ] + agg = bench.aggregate(scored) + assert agg["en_scores"]["text_recognition"] == {"avg": 0.5, "count": 2} + assert agg["en_scores"]["mathematical_calculation"]["avg"] == 1.0 + assert agg["en_scores"]["text_spotting"]["count"] == 0 # ignored one didn't count + # overall = mean of the two non-empty category means + assert agg["en_overall"] == pytest.approx((0.5 + 1.0) / 2) + + +@pytest.mark.parametrize("tag", ARCHIVED) +def test_aggregation_parity_with_archived_run(tag): + scored_path = RESULTS / f"{tag}_scored.json" + metrics_path = RESULTS / f"{tag}_metrics.json" + if not (scored_path.exists() and metrics_path.exists()): + pytest.skip(f"archived run {tag} not present") + + scored = json.loads(scored_path.read_text()) + archived = json.loads(metrics_path.read_text()) + agg = bench.aggregate(scored) + + assert agg["en_overall"] == pytest.approx(archived["en_overall"]) + assert agg["cn_overall"] == pytest.approx(archived["cn_overall"]) + for cat, v in archived["en_scores"].items(): + assert agg["en_scores"][cat]["count"] == v["count"] + assert agg["en_scores"][cat]["avg"] == pytest.approx(v["avg"]) From 650ee04752e88b5161a01825c98920e905d15c15 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 09:00:07 +0530 Subject: [PATCH 13/38] feat(bench_core): migrate olmOCR onto the abstraction; retire 6 drivers + 6 runners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit benchmarks/olmocr/harness.py routes per-page inference through bench_core and reuses the external olmocr.bench scorer: score() materializes .md files from the run records, invokes the scorer, tees stdout to logs/olmocr_.log (what report_scores reads), and parses the overall out. Named harness.py to avoid the vendored benchmarks/olmocr/bench/ package. CLI passes target to score() when its signature accepts it. Retires olmocr_bench{,_fireworks,_gemini,_gemini_pro_31,_grok,_openai_mini}.py and the matching run_*.py. Kept: olmocr_bench_reducto.py + run_reducto.py (Reducto is a bespoke document API, not a chat model — like ob_det_api). No offline score-replay (the scorer needs the dataset + prints to a log); parity is by-construction (identical prompt/PNG render/null-normalization) plus a log-parser test against the archived gemini run. 153 tests pass. --- README.md | 22 +- bench_core/cli.py | 7 +- .../olmocr/bench/runners/run_fireworks.py | 99 ------- benchmarks/olmocr/bench/runners/run_gemini.py | 102 ------- .../olmocr/bench/runners/run_gemini_pro_31.py | 72 ----- benchmarks/olmocr/bench/runners/run_grok.py | 76 ----- .../olmocr/bench/runners/run_interfaze.py | 93 ------ .../olmocr/bench/runners/run_openai_mini.py | 79 ----- benchmarks/olmocr/harness.py | 213 +++++++++++++ benchmarks/olmocr/olmocr_bench.py | 248 ---------------- benchmarks/olmocr/olmocr_bench_fireworks.py | 279 ----------------- benchmarks/olmocr/olmocr_bench_gemini.py | 280 ------------------ .../olmocr/olmocr_bench_gemini_pro_31.py | 202 ------------- benchmarks/olmocr/olmocr_bench_grok.py | 202 ------------- benchmarks/olmocr/olmocr_bench_openai_mini.py | 202 ------------- scripts/run_fireworks_suite.sh | 6 +- tests/test_olmocr_bench.py | 39 +++ 17 files changed, 271 insertions(+), 1950 deletions(-) delete mode 100644 benchmarks/olmocr/bench/runners/run_fireworks.py delete mode 100644 benchmarks/olmocr/bench/runners/run_gemini.py delete mode 100644 benchmarks/olmocr/bench/runners/run_gemini_pro_31.py delete mode 100644 benchmarks/olmocr/bench/runners/run_grok.py delete mode 100644 benchmarks/olmocr/bench/runners/run_interfaze.py delete mode 100644 benchmarks/olmocr/bench/runners/run_openai_mini.py create mode 100644 benchmarks/olmocr/harness.py delete mode 100644 benchmarks/olmocr/olmocr_bench.py delete mode 100644 benchmarks/olmocr/olmocr_bench_fireworks.py delete mode 100644 benchmarks/olmocr/olmocr_bench_gemini.py delete mode 100644 benchmarks/olmocr/olmocr_bench_gemini_pro_31.py delete mode 100644 benchmarks/olmocr/olmocr_bench_grok.py delete mode 100644 benchmarks/olmocr/olmocr_bench_openai_mini.py create mode 100644 tests/test_olmocr_bench.py diff --git a/README.md b/README.md index 9ab86cb..a638dae 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ audio input, so it runs on every benchmark here: ```bash uv run python -m bench_core run --target inkling --benchmark ocrbench_v2 -uv run -m benchmarks.olmocr.olmocr_bench_fireworks +uv run python -m bench_core run --target inkling --benchmark olmocr uv run python -m bench_core run --target inkling --benchmark gpqa uv run python -m bench_core run --target inkling --benchmark mmmlu uv run python -m bench_core run --target inkling --benchmark mmmu_pro --variant standard @@ -155,19 +155,13 @@ uv run -m benchmarks.ocrbench_v2.ocrbench_v2 --evaluate-only Links: [repo](https://github.com/allenai/olmocr/tree/main/olmocr/bench) · [dataset](https://huggingface.co/datasets/allenai/olmOCR-bench) ```bash -# Interfaze -uv run -m benchmarks.olmocr.olmocr_bench - -# Per-provider runners -uv run -m benchmarks.olmocr.olmocr_bench_openai_mini -uv run -m benchmarks.olmocr.olmocr_bench_gemini_pro_31 -uv run -m benchmarks.olmocr.olmocr_bench_grok -uv run -m benchmarks.olmocr.olmocr_bench_fireworks - -# Useful flags -uv run -m benchmarks.olmocr.olmocr_bench --sample # tiny sample dataset -uv run -m benchmarks.olmocr.olmocr_bench --generate-only # predictions only -uv run -m benchmarks.olmocr.olmocr_bench --skip-generation # evaluation only +# Any target (renders each PDF page, scores via the external olmocr.bench scorer; +# results land in logs/olmocr_.log, which the report reads) +uv run python -m bench_core run --target gemini-3.7-flash --benchmark olmocr +uv run python -m bench_core run --target inkling --benchmark olmocr + +# Reducto (a bespoke document-parsing API, not a chat model) keeps its own runner +uv run -m benchmarks.olmocr.olmocr_bench_reducto ``` --- diff --git a/bench_core/cli.py b/bench_core/cli.py index b2574e9..dd142a3 100644 --- a/bench_core/cli.py +++ b/bench_core/cli.py @@ -5,6 +5,7 @@ import argparse import asyncio import importlib +import inspect from bench_core.config import ( build_adapter, @@ -23,6 +24,7 @@ "mmmu_pro": "benchmarks.mmmu_pro.bench", "refcoco": "benchmarks.obj_detection.bench", "ocrbench_v2": "benchmarks.ocrbench_v2.bench", + "olmocr": "benchmarks.olmocr.harness", } @@ -79,7 +81,10 @@ def cmd_run(args) -> None: ) ) - metrics = bench.score(store.load_responses(), samples) + score_kwargs = {} + if "target" in inspect.signature(bench.score).parameters: + score_kwargs["target"] = target + metrics = bench.score(store.load_responses(), samples, **score_kwargs) metrics.update( { "benchmark": result_name, diff --git a/benchmarks/olmocr/bench/runners/run_fireworks.py b/benchmarks/olmocr/bench/runners/run_fireworks.py deleted file mode 100644 index 1cbbe87..0000000 --- a/benchmarks/olmocr/bench/runners/run_fireworks.py +++ /dev/null @@ -1,99 +0,0 @@ -import os - -from openai import OpenAI - -from benchmarks.olmocr.data.renderpdf import render_pdf_to_base64png - -_client = None - -INKLING = "accounts/fireworks/models/inkling" - - -def _get_client() -> OpenAI: - """Host is chosen by BENCH_OLMOCR_PROVIDER (fireworks|openrouter). The hosts - are not equivalent at the same nominal reasoning setting — see README.""" - global _client - if _client is None: - if os.getenv("BENCH_OLMOCR_PROVIDER") == "openrouter": - api_key = os.getenv("OPENROUTER_API_KEY") or os.getenv("openrouter_api_key") - if not api_key: - raise SystemExit("OPENROUTER_API_KEY not set") - _client = OpenAI( - api_key=api_key, base_url="https://openrouter.ai/api/v1" - ) - return _client - # .env in this repo may spell the key lowercase; accept either casing. - api_key = os.getenv("FIREWORKS_API_KEY") or os.getenv("fireworks_api_key") - if not api_key: - raise SystemExit( - "FIREWORKS_API_KEY not set — get it from " - "https://app.fireworks.ai/settings/users/api-keys" - ) - _client = OpenAI( - api_key=api_key, base_url="https://api.fireworks.ai/inference/v1" - ) - return _client - - -def run_fireworks( - pdf_path: str, - page_num: int = 1, - model: str = INKLING, - target_longest_image_dim: int = 2048, -) -> str: - """Convert a PDF page to markdown via a Fireworks-hosted model (Inkling). - - reasoning_effort='none' is Inkling's floor — it still thinks, so the page - transcription is preceded by reasoning tokens we pay for but discard.""" - image_base64 = render_pdf_to_base64png( - pdf_path, page_num=page_num, target_longest_image_dim=target_longest_image_dim - ) - - client = _get_client() - - prompt = ( - "Below is the image of one page of a PDF document. " - "Just return the plain text representation of this document as if you were reading it naturally.\n" - "Turn equations into LaTeX using \\( \\) for inline math and \\[ \\] for display math. " - "Never describe equations in words — always use LaTeX notation. " - "Turn tables into markdown format.\n" - "Remove the headers and footers completely — do not include any text " - "that appears at the very top or very bottom of the page outside the main body content. " - "This includes page numbers, journal names, author names in running headers, " - "copyright lines, DOI lines, citation requests, institutional addresses in margins, " - "and download dates. Keep references and footnotes that are part of the body.\n" - "For multi-column layouts, read each column top to bottom before moving to the next.\n" - "Read any natural handwriting.\n" - "This is likely one page out of several in the document, so be sure to preserve " - "any sentences that come from the previous page, or continue onto the next page, exactly as they are.\n" - "If there is no text at all that you think you should read, you can output null.\n" - "Do not hallucinate." - ) - - response = client.chat.completions.create( - model=model, - messages=[ - { - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - { - "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{image_base64}"}, - }, - ], - } - ], - max_tokens=20000, - **( - {"extra_body": {"reasoning": {"effort": os.getenv("BENCH_OLMOCR_EFFORT", "none")}}} - if os.getenv("BENCH_OLMOCR_PROVIDER") == "openrouter" - else {"reasoning_effort": os.getenv("BENCH_OLMOCR_EFFORT", "none")} - ), - ) - - assert len(response.choices) > 0 - raw = response.choices[0].message.content - if raw is None or raw.strip().lower() in ("null", "none", "n/a", ""): - return "" - return raw diff --git a/benchmarks/olmocr/bench/runners/run_gemini.py b/benchmarks/olmocr/bench/runners/run_gemini.py deleted file mode 100644 index d038bea..0000000 --- a/benchmarks/olmocr/bench/runners/run_gemini.py +++ /dev/null @@ -1,102 +0,0 @@ -import base64 -import os - -from google import genai -from google.genai import types - -from benchmarks.olmocr.data.renderpdf import render_pdf_to_base64png - -_client = None - -DEFAULT_MODEL = "gemini-3.7-flash" - - -def _get_client(): - global _client - if _client is None: - api_key = ( - os.getenv("GEMINI_API_KEY") - or os.getenv("GEMINI_KEY") - or os.getenv("GOOGLE_API_KEY") - ) - if not api_key: - raise SystemExit( - "GEMINI_API_KEY not set — get it from https://aistudio.google.com/apikey" - ) - # Cached deliberately: building a client per call gets it collected - # mid-flight ("Cannot send a request, as the client has been closed"). - _client = genai.Client(api_key=api_key) - return _client - - -def _thinking_floor(model: str) -> str: - """Lowest thinking_level the model accepts. - - gemini-3.7+ flash rejects MINIMAL server-side ("Thinking level MINIMAL is - not supported for this model") and the SDK enum has no off/none, so "low" is - its floor. Older 3.x flash still takes "minimal"; Pro models reject - "minimal" too. Override with BENCH_GEMINI_FLOOR. - """ - override = os.getenv("BENCH_GEMINI_FLOOR") - if override: - return override - m = model.lower() - if "pro" in m or any(v in m for v in ("3.7", "3.8", "3.9", "4.")): - return "low" - return "minimal" - - -def run_gemini( - pdf_path: str, - page_num: int = 1, - model: str = DEFAULT_MODEL, - target_longest_image_dim: int = 2048, -) -> str: - """Convert a PDF page to markdown via any vision-capable Gemini model. - - Generalises run_gemini_pro_31, which pinned both the model and its thinking - config. Same prompt, so scores stay comparable across models. - """ - image_base64 = render_pdf_to_base64png( - pdf_path, page_num=page_num, target_longest_image_dim=target_longest_image_dim - ) - image_bytes = base64.b64decode(image_base64) - - client = _get_client() - - prompt = ( - "Below is the image of one page of a PDF document. " - "Just return the plain text representation of this document as if you were reading it naturally.\n" - "Turn equations into LaTeX using \\( \\) for inline math and \\[ \\] for display math. " - "Never describe equations in words — always use LaTeX notation. " - "Turn tables into markdown format.\n" - "Remove the headers and footers completely — do not include any text " - "that appears at the very top or very bottom of the page outside the main body content. " - "This includes page numbers, journal names, author names in running headers, " - "copyright lines, DOI lines, citation requests, institutional addresses in margins, " - "and download dates. Keep references and footnotes that are part of the body.\n" - "For multi-column layouts, read each column top to bottom before moving to the next.\n" - "Read any natural handwriting.\n" - "This is likely one page out of several in the document, so be sure to preserve " - "any sentences that come from the previous page, or continue onto the next page, exactly as they are.\n" - "If there is no text at all that you think you should read, you can output null.\n" - "Do not hallucinate." - ) - - image_part = types.Part.from_bytes(data=image_bytes, mime_type="image/png") - - config = types.GenerateContentConfig( - temperature=0.0, - thinking_config=types.ThinkingConfig(thinking_level=_thinking_floor(model)), - ) - - response = client.models.generate_content( - model=model, - contents=[prompt, image_part], - config=config, - ) - - raw = response.text if hasattr(response, "text") else None - if raw is None or raw.strip().lower() in ("null", "none", "n/a", ""): - return "" - return raw diff --git a/benchmarks/olmocr/bench/runners/run_gemini_pro_31.py b/benchmarks/olmocr/bench/runners/run_gemini_pro_31.py deleted file mode 100644 index bdb83b5..0000000 --- a/benchmarks/olmocr/bench/runners/run_gemini_pro_31.py +++ /dev/null @@ -1,72 +0,0 @@ -import os - -from google import genai -from google.genai import types - -from benchmarks.olmocr.data.renderpdf import render_pdf_to_base64png - -_client = None - - -def _get_client(): - global _client - if _client is None: - api_key = os.getenv("GEMINI_API_KEY") - if not api_key: - raise SystemExit( - "GEMINI_API_KEY not set — get it from https://aistudio.google.com/apikey" - ) - _client = genai.Client(api_key=api_key) - return _client - - -def run_gemini_pro_31( - pdf_path: str, - page_num: int = 1, - model: str = "gemini-3.1-pro-preview", - target_longest_image_dim: int = 2048, -) -> str: - """Convert a PDF page to markdown via Gemini 3.1 Pro Preview (vision).""" - import base64 - - image_base64 = render_pdf_to_base64png( - pdf_path, page_num=page_num, target_longest_image_dim=target_longest_image_dim - ) - image_bytes = base64.b64decode(image_base64) - - client = _get_client() - - prompt = ( - "Below is the image of one page of a PDF document. " - "Just return the plain text representation of this document as if you were reading it naturally.\n" - "Turn equations into LaTeX using \\( \\) for inline math and \\[ \\] for display math. " - "Never describe equations in words — always use LaTeX notation. " - "Turn tables into markdown format.\n" - "Remove the headers and footers completely — do not include any text " - "that appears at the very top or very bottom of the page outside the main body content. " - "This includes page numbers, journal names, author names in running headers, " - "copyright lines, DOI lines, citation requests, institutional addresses in margins, " - "and download dates. Keep references and footnotes that are part of the body.\n" - "For multi-column layouts, read each column top to bottom before moving to the next.\n" - "Read any natural handwriting.\n" - "This is likely one page out of several in the document, so be sure to preserve " - "any sentences that come from the previous page, or continue onto the next page, exactly as they are.\n" - "If there is no text at all that you think you should read, you can output null.\n" - "Do not hallucinate." - ) - - image_part = types.Part.from_bytes(data=image_bytes, mime_type="image/png") - - # Pro models reject thinking_budget=0 — let API use default thinking. - config = types.GenerateContentConfig() - - response = client.models.generate_content( - model=model, - contents=[prompt, image_part], - config=config, - ) - - raw = response.text if hasattr(response, "text") else None - if raw is None or raw.strip().lower() in ("null", "none", "n/a", ""): - return "" - return raw diff --git a/benchmarks/olmocr/bench/runners/run_grok.py b/benchmarks/olmocr/bench/runners/run_grok.py deleted file mode 100644 index 64693df..0000000 --- a/benchmarks/olmocr/bench/runners/run_grok.py +++ /dev/null @@ -1,76 +0,0 @@ -import os - -from openai import OpenAI - -from benchmarks.olmocr.data.renderpdf import render_pdf_to_base64png - -_client = None - - -def _get_client() -> OpenAI: - global _client - if _client is None: - api_key = os.getenv("OPENROUTER_API_KEY") - if not api_key: - raise SystemExit( - "OPENROUTER_API_KEY not set — get it from https://openrouter.ai/keys" - ) - _client = OpenAI(api_key=api_key, base_url="https://openrouter.ai/api/v1") - return _client - - -def run_grok( - pdf_path: str, - page_num: int = 1, - model: str = "x-ai/grok-4.3", - target_longest_image_dim: int = 2048, -) -> str: - """Convert a PDF page to markdown via Grok 4.3 with reasoning_effort='low'.""" - image_base64 = render_pdf_to_base64png( - pdf_path, page_num=page_num, target_longest_image_dim=target_longest_image_dim - ) - - client = _get_client() - - prompt = ( - "Below is the image of one page of a PDF document. " - "Just return the plain text representation of this document as if you were reading it naturally.\n" - "Turn equations into LaTeX using \\( \\) for inline math and \\[ \\] for display math. " - "Never describe equations in words — always use LaTeX notation. " - "Turn tables into markdown format.\n" - "Remove the headers and footers completely — do not include any text " - "that appears at the very top or very bottom of the page outside the main body content. " - "This includes page numbers, journal names, author names in running headers, " - "copyright lines, DOI lines, citation requests, institutional addresses in margins, " - "and download dates. Keep references and footnotes that are part of the body.\n" - "For multi-column layouts, read each column top to bottom before moving to the next.\n" - "Read any natural handwriting.\n" - "This is likely one page out of several in the document, so be sure to preserve " - "any sentences that come from the previous page, or continue onto the next page, exactly as they are.\n" - "If there is no text at all that you think you should read, you can output null.\n" - "Do not hallucinate." - ) - - response = client.chat.completions.create( - model=model, - messages=[ - { - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - { - "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{image_base64}"}, - }, - ], - } - ], - reasoning_effort="low", - max_completion_tokens=20000, - ) - - assert len(response.choices) > 0 - raw = response.choices[0].message.content - if raw is None or raw.strip().lower() in ("null", "none", "n/a", ""): - return "" - return raw diff --git a/benchmarks/olmocr/bench/runners/run_interfaze.py b/benchmarks/olmocr/bench/runners/run_interfaze.py deleted file mode 100644 index 7506b6d..0000000 --- a/benchmarks/olmocr/bench/runners/run_interfaze.py +++ /dev/null @@ -1,93 +0,0 @@ -import os - -from openai import OpenAI - -from benchmarks.olmocr.data.renderpdf import render_pdf_to_base64png - -def run_interfaze( - pdf_path: str, - page_num: int = 1, - model: str = "interfaze-beta", - temperature: float = 0.1, - target_longest_image_dim: int = 2048, -) -> str: - """ - Convert a page of a PDF file to markdown using the Interfaze API. - - Args: - pdf_path: The local path to the PDF file. - page_num: The page number to process (starting from 1). - model: The Interfaze model to use. - temperature: The temperature parameter for generation. - target_longest_image_dim: Target longest image dimension for rendering. - - Returns: - The OCR result in markdown format. - """ - image_base64 = render_pdf_to_base64png( - pdf_path, page_num=page_num, target_longest_image_dim=target_longest_image_dim - ) - - api_key = os.getenv("INTERFAZE_API_KEY") - if not api_key: - raise SystemExit( - "You must set INTERFAZE_API_KEY - get it from https://interfaze.ai/dashboard" - ) - - client = OpenAI( - base_url=os.getenv("OPENAI_BASE_URL", "https://api.interfaze.ai/v1"), - api_key=api_key, - ) - - - prompt = ( - "Below is the image of one page of a PDF document. " - "Just return the plain text representation of this document as if you were reading it naturally.\n" - "Turn equations into LaTeX using \\( \\) for inline math and \\[ \\] for display math. " - "Never describe equations in words — always use LaTeX notation. " - "Turn tables into markdown format.\n" - "Remove the headers and footers completely — do not include any text " - "that appears at the very top or very bottom of the page outside the main body content. " - "This includes page numbers, journal names, author names in running headers, " - "copyright lines, DOI lines, citation requests, institutional addresses in margins, " - "and download dates. Keep references and footnotes that are part of the body.\n" - "For multi-column layouts, read each column top to bottom before moving to the next.\n" - "Read any natural handwriting.\n" - "This is likely one page out of several in the document, so be sure to preserve " - "any sentences that come from the previous page, or continue onto the next page, exactly as they are.\n" - "If there is no text at all that you think you should read, you can output null.\n" - "Do not hallucinate." - ) - - - - response = client.chat.completions.create( - model=model, - messages=[ - { - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - { - "type": "image_url", - "image_url": { - "url": f"data:image/png;base64,{image_base64}" - }, - }, - ], - } - ], - temperature=temperature, - max_completion_tokens=20000, - ) - - assert len(response.choices) > 0 - assert response.choices[0].finish_reason == "stop" - - raw = response.choices[0].message.content - - # Handle blank pages - model may return "null" or similar for empty pages - if raw is None or raw.strip().lower() in ("null", "none", "n/a", ""): - return "" - - return raw diff --git a/benchmarks/olmocr/bench/runners/run_openai_mini.py b/benchmarks/olmocr/bench/runners/run_openai_mini.py deleted file mode 100644 index 2eb1a7f..0000000 --- a/benchmarks/olmocr/bench/runners/run_openai_mini.py +++ /dev/null @@ -1,79 +0,0 @@ -import os - -from openai import OpenAI - -from benchmarks.olmocr.data.renderpdf import render_pdf_to_base64png - -# Direct OpenAI client (not commons_openai, since commons module-level read of -# OPENAI_API_KEY would conflict with how this file is imported lazily). -_client = None - - -def _get_client() -> OpenAI: - global _client - if _client is None: - api_key = os.getenv("OPENAI_API_KEY") - if not api_key: - raise SystemExit( - "OPENAI_API_KEY not set — get it from https://platform.openai.com/api-keys" - ) - base_url = os.getenv("OPENAI_API_BASE_URL", "https://api.openai.com/v1") - _client = OpenAI(api_key=api_key, base_url=base_url) - return _client - - -def run_openai_mini( - pdf_path: str, - page_num: int = 1, - model: str = "gpt-5.4-mini", - target_longest_image_dim: int = 2048, -) -> str: - """Convert a PDF page to markdown via OpenAI gpt-5.4-mini (vision).""" - image_base64 = render_pdf_to_base64png( - pdf_path, page_num=page_num, target_longest_image_dim=target_longest_image_dim - ) - - client = _get_client() - - prompt = ( - "Below is the image of one page of a PDF document. " - "Just return the plain text representation of this document as if you were reading it naturally.\n" - "Turn equations into LaTeX using \\( \\) for inline math and \\[ \\] for display math. " - "Never describe equations in words — always use LaTeX notation. " - "Turn tables into markdown format.\n" - "Remove the headers and footers completely — do not include any text " - "that appears at the very top or very bottom of the page outside the main body content. " - "This includes page numbers, journal names, author names in running headers, " - "copyright lines, DOI lines, citation requests, institutional addresses in margins, " - "and download dates. Keep references and footnotes that are part of the body.\n" - "For multi-column layouts, read each column top to bottom before moving to the next.\n" - "Read any natural handwriting.\n" - "This is likely one page out of several in the document, so be sure to preserve " - "any sentences that come from the previous page, or continue onto the next page, exactly as they are.\n" - "If there is no text at all that you think you should read, you can output null.\n" - "Do not hallucinate." - ) - - response = client.chat.completions.create( - model=model, - messages=[ - { - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - { - "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{image_base64}"}, - }, - ], - } - ], - reasoning_effort="none", - max_completion_tokens=20000, - ) - - assert len(response.choices) > 0 - raw = response.choices[0].message.content - if raw is None or raw.strip().lower() in ("null", "none", "n/a", ""): - return "" - return raw diff --git a/benchmarks/olmocr/harness.py b/benchmarks/olmocr/harness.py new file mode 100644 index 0000000..488e70a --- /dev/null +++ b/benchmarks/olmocr/harness.py @@ -0,0 +1,213 @@ +"""olmOCR-bench: PDF page -> markdown, scored by the external olmocr.bench scorer. + +Special-cased: the scorer reads a directory of per-page .md files and prints its +table to stdout (no metrics.json), so `score` materializes the .md files from the +run records, invokes the scorer, tees stdout to logs/olmocr_.log (what +report_scores reads), and parses the overall out of it. +""" + +from __future__ import annotations + +import json +import os +import re +import sys +from pathlib import Path + +from bench_core.request import ImagePart, Message, ReasoningSpec, Request, TextPart +from bench_core.results import model_slug + +NAME = "olmocr" +ID_KEY = "id" +PRIMARY_METRIC = "overall" +DEFAULTS = {"reasoning": "off", "rate_limit": 25, "max_in_flight": 8} + +_HF_REPO = "allenai/olmOCR-bench" +_FULL_DATA_DIR = Path(__file__).resolve().parent / "bench" / "full_data" +_LOGS_DIR = Path(__file__).resolve().parent.parent.parent / "logs" +_SPLITS = [ + "arxiv_math", + "headers_footers", + "long_tiny_text", + "multi_column", + "old_scans", + "old_scans_math", + "table_tests", +] +_MAX_TOKENS = 20000 + +PROMPT = ( + "Below is the image of one page of a PDF document. " + "Just return the plain text representation of this document as if you were reading it naturally.\n" + "Turn equations into LaTeX using \\( \\) for inline math and \\[ \\] for display math. " + "Never describe equations in words — always use LaTeX notation. " + "Turn tables into markdown format.\n" + "Remove the headers and footers completely — do not include any text " + "that appears at the very top or very bottom of the page outside the main body content. " + "This includes page numbers, journal names, author names in running headers, " + "copyright lines, DOI lines, citation requests, institutional addresses in margins, " + "and download dates. Keep references and footnotes that are part of the body.\n" + "For multi-column layouts, read each column top to bottom before moving to the next.\n" + "Read any natural handwriting.\n" + "This is likely one page out of several in the document, so be sure to preserve " + "any sentences that come from the previous page, or continue onto the next page, exactly as they are.\n" + "If there is no text at all that you think you should read, you can output null.\n" + "Do not hallucinate." +) + +_NULL = ("null", "none", "n/a", "") + + +def _download() -> Path: + from huggingface_hub import hf_hub_download + + data_dir = _FULL_DATA_DIR + pdf_dir = data_dir / "pdfs" + all_pdfs = set() + for split in _SPLITS: + dest = data_dir / f"{split}.jsonl" + if not dest.exists(): + src = hf_hub_download( + _HF_REPO, f"bench_data/{split}.jsonl", repo_type="dataset" + ) + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(Path(src).read_text()) + for line in dest.read_text().splitlines(): + if line.strip(): + all_pdfs.add(json.loads(line)["pdf"]) + for pdf_rel in sorted(all_pdfs): + local = pdf_dir / pdf_rel + if local.exists(): + continue + local.parent.mkdir(parents=True, exist_ok=True) + src = hf_hub_download( + _HF_REPO, f"bench_data/pdfs/{pdf_rel}", repo_type="dataset" + ) + os.symlink(src, str(local)) + return data_dir + + +def load_samples(sample_size: int | None = None) -> list[dict]: + data_dir = _download() + pairs = set() + for jf in data_dir.glob("*.jsonl"): + for line in jf.read_text().splitlines(): + if line.strip(): + t = json.loads(line) + pairs.add((t["pdf"], t["page"])) + samples = [] + for pdf_rel, page in sorted(pairs): + pdf_path = data_dir / "pdfs" / pdf_rel + if not pdf_path.exists(): + continue + base = os.path.splitext(os.path.basename(pdf_rel))[0] + parent = os.path.dirname(pdf_rel) + out_rel = ( + f"{parent}/{base}_pg{page}_repeat1.md" + if parent + else f"{base}_pg{page}_repeat1.md" + ) + samples.append( + { + "id": f"{pdf_rel}#{page}", + "pdf_path": str(pdf_path), + "page": page, + "out_rel": out_rel, + } + ) + return samples[:sample_size] if sample_size else samples + + +def build_request(sample: dict, mode: str) -> Request: + from benchmarks.olmocr.data.renderpdf import render_pdf_to_base64png + + b64 = render_pdf_to_base64png( + sample["pdf_path"], page_num=sample["page"], target_longest_image_dim=2048 + ) + import base64 + + png = base64.b64decode(b64) + return Request( + [Message("user", [TextPart(PROMPT), ImagePart(png, "image/png")])], + reasoning=ReasoningSpec(mode), + temperature=0.0, + max_tokens=_MAX_TOKENS, + ) + + +def parse(response, sample) -> str: + raw = response.text or "" + return "" if raw.strip().lower() in _NULL else raw + + +class _Tee: + def __init__(self, *streams): + self.streams = streams + + def write(self, data): + for s in self.streams: + s.write(data) + s.flush() + + def flush(self): + for s in self.streams: + s.flush() + + +def score(records: list[dict], samples: list[dict], target=None) -> dict: + candidate = model_slug(target.name) if target is not None else "candidate" + by_id = {s["id"]: s for s in samples} + out_root = _FULL_DATA_DIR / candidate + # materialize per-page .md from the run records (the scorer reads these) + for r in records: + s = by_id.get(r["id"]) + if s is None: + continue + md_path = out_root / s["out_rel"] + md_path.parent.mkdir(parents=True, exist_ok=True) + md_path.write_text(r.get("prediction") or "", encoding="utf-8") + + from olmocr.bench.benchmark import main as bench_main + + _LOGS_DIR.mkdir(parents=True, exist_ok=True) + log_path = _LOGS_DIR / f"olmocr_{candidate}.log" + import io + + buf = io.StringIO() + argv, real = sys.argv, sys.stdout + sys.argv = [ + "benchmark", + "--dir", + str(_FULL_DATA_DIR), + "--candidate", + candidate, + "--force", + ] + with open(log_path, "w", encoding="utf-8") as fh: + sys.stdout = _Tee(real, fh, buf) + try: + bench_main() + finally: + sys.stdout, sys.argv = real, argv + + return { + "candidate": candidate, + "log": str(log_path), + **_parse_scores(buf.getvalue()), + } + + +def _parse_scores(text: str) -> dict: + text = text.replace("\r", "\n") + head = re.search( + r"^(\S+)\s*:\s*Average Score:\s*([\d.]+)%\s*±\s*([\d.]+)%", text, re.MULTILINE + ) + overall = float(head.group(2)) if head else None + splits = { + m.group(1): float(m.group(2)) + for m in re.finditer(r"^\s+(\w+)\.jsonl\s*:\s*([\d.]+)%", text, re.MULTILINE) + } + base = re.search(r"^\s+baseline\s*:\s*([\d.]+)%", text, re.MULTILINE) + if base: + splits["baseline"] = float(base.group(1)) + return {"overall": overall, "splits": splits} diff --git a/benchmarks/olmocr/olmocr_bench.py b/benchmarks/olmocr/olmocr_bench.py deleted file mode 100644 index fe48770..0000000 --- a/benchmarks/olmocr/olmocr_bench.py +++ /dev/null @@ -1,248 +0,0 @@ -""" -OlmOCR Benchmark for Interfaze. - -Downloads the full olmOCR-bench dataset from HuggingFace, processes all PDFs -through the Interfaze API, and evaluates against the test suite. - -Usage: - uv run -m benchmarks.olmocr.olmocr_bench # full run - uv run -m benchmarks.olmocr.olmocr_bench --sample # sample data only - uv run -m benchmarks.olmocr.olmocr_bench --skip-generation # evaluate only - uv run -m benchmarks.olmocr.olmocr_bench --generate-only # generate only -""" - -import argparse -import asyncio -import json -import os -import sys -from pathlib import Path - -from dotenv import load_dotenv -from huggingface_hub import hf_hub_download -from pypdf import PdfReader -from tqdm.asyncio import tqdm_asyncio - -load_dotenv() - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -SAMPLE_DATA_DIR = Path(__file__).resolve().parent / "bench" / "sample_data" -FULL_DATA_DIR = Path(__file__).resolve().parent / "bench" / "full_data" -CANDIDATE_NAME = "interfaze" -RATE_LIMIT = 25 -MAX_RETRIES = 3 - -HF_REPO = "allenai/olmOCR-bench" -SPLITS = [ - "arxiv_math", - "headers_footers", - "long_tiny_text", - "multi_column", - "old_scans", - "old_scans_math", - "table_tests", -] - -sys.path.insert(0, str(PROJECT_ROOT)) -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def download_full_dataset(): - """Download JSONL test files and PDFs from HuggingFace into full_data/.""" - data_dir = FULL_DATA_DIR - pdf_dir = data_dir / "pdfs" - - # Download JSONL files - all_pdfs = set() - for split in SPLITS: - jsonl_dest = data_dir / f"{split}.jsonl" - if jsonl_dest.exists(): - print(f" {split}.jsonl already exists, loading PDF list...") - with open(jsonl_dest) as f: - tests = [json.loads(l) for l in f if l.strip()] - else: - print(f" Downloading {split}.jsonl...") - src = hf_hub_download(HF_REPO, f"bench_data/{split}.jsonl", repo_type="dataset") - with open(src) as f: - tests = [json.loads(l) for l in f if l.strip()] - # Copy to our data dir - data_dir.mkdir(parents=True, exist_ok=True) - with open(jsonl_dest, "w") as f: - for t in tests: - f.write(json.dumps(t) + "\n") - - print(f" {split}: {len(tests)} tests") - for t in tests: - all_pdfs.add(t["pdf"]) - - print(f"\n Total unique PDFs to download: {len(all_pdfs)}") - - # Download PDFs - downloaded = 0 - skipped = 0 - for pdf_rel in sorted(all_pdfs): - local_path = pdf_dir / pdf_rel - if local_path.exists(): - skipped += 1 - continue - local_path.parent.mkdir(parents=True, exist_ok=True) - try: - src = hf_hub_download(HF_REPO, f"bench_data/pdfs/{pdf_rel}", repo_type="dataset") - # Symlink to HF cache to save disk space - os.symlink(src, str(local_path)) - downloaded += 1 - except Exception as e: - print(f" Failed to download {pdf_rel}: {e}") - - print(f" PDFs: {downloaded} downloaded, {skipped} already existed") - return data_dir - - -async def process_page( - pdf_path: str, - page_num: int, - output_path: str, - rate_limiter: RateLimiter, -) -> bool: - """Process a single PDF page through Interfaze and save the markdown output.""" - from olmocr.bench.runners.run_interfaze import run_interfaze - - for attempt in range(MAX_RETRIES): - await rate_limiter.acquire() - try: - result = await asyncio.to_thread(run_interfaze, pdf_path, page_num) - os.makedirs(os.path.dirname(output_path), exist_ok=True) - with open(output_path, "w", encoding="utf-8") as f: - f.write(result) - return True - except Exception as e: - if attempt < MAX_RETRIES - 1: - await asyncio.sleep(2**attempt) - else: - print(f"Failed after {MAX_RETRIES} attempts: {pdf_path} page {page_num}: {e}") - return False - - -async def generate_outputs(data_dir: Path): - """Generate markdown outputs for all PDFs using Interfaze.""" - pdf_folder = data_dir / "pdfs" - output_folder = data_dir / CANDIDATE_NAME - - # Collect all unique (pdf, page) pairs from JSONL test files - pdf_pages = set() - for jsonl_file in data_dir.glob("*.jsonl"): - with open(jsonl_file) as f: - for line in f: - line = line.strip() - if not line: - continue - t = json.loads(line) - pdf_pages.add((t["pdf"], t["page"])) - - print(f"Found {len(pdf_pages)} unique (pdf, page) pairs to process") - - rate_limiter = RateLimiter(RATE_LIMIT) - tasks = [] - - for pdf_rel, page in sorted(pdf_pages): - pdf_path = str(pdf_folder / pdf_rel) - if not os.path.exists(pdf_path): - continue - - base_name = os.path.splitext(os.path.basename(pdf_rel))[0] - parent_dir = os.path.dirname(pdf_rel) - md_filename = f"{base_name}_pg{page}_repeat1.md" - - if parent_dir: - out_path = str(output_folder / parent_dir / md_filename) - else: - out_path = str(output_folder / md_filename) - - if os.path.exists(out_path): - continue - - tasks.append(process_page(pdf_path, page, out_path, rate_limiter)) - - if not tasks: - print("All outputs already exist, skipping generation.") - return True - - print(f"Processing {len(tasks)} pages...") - results = await tqdm_asyncio.gather(*tasks, desc="Generating markdown outputs") - num_success = sum(1 for r in results if r) - num_failed = len(results) - num_success - print(f"Done: {num_success} succeeded, {num_failed} failed") - - return num_failed == 0 - - -def run_evaluation(data_dir: Path): - """Run the olmocr benchmark evaluation for the Interfaze candidate.""" - from olmocr.bench.benchmark import main as bench_main - - sys.argv = [ - "benchmark", - "--dir", str(data_dir), - "--candidate", CANDIDATE_NAME, - "--force", - ] - bench_main() - - -async def main(): - parser = argparse.ArgumentParser(description="Run OlmOCR benchmark with Interfaze") - parser.add_argument( - "--sample", - action="store_true", - help="Use sample data only (small subset for testing)", - ) - parser.add_argument( - "--skip-generation", - action="store_true", - help="Skip markdown generation, only run evaluation", - ) - parser.add_argument( - "--generate-only", - action="store_true", - help="Only generate markdown outputs, skip evaluation", - ) - args = parser.parse_args() - - if args.sample: - data_dir = SAMPLE_DATA_DIR - print("=== Using sample data ===") - else: - print("=== Downloading full olmOCR-bench dataset from HuggingFace ===") - data_dir = download_full_dataset() - - if not args.skip_generation: - print("\n=== Generating Interfaze outputs ===") - await generate_outputs(data_dir) - - if not args.generate_only: - print("\n=== Running OlmOCR Benchmark Evaluation ===") - run_evaluation(data_dir) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/benchmarks/olmocr/olmocr_bench_fireworks.py b/benchmarks/olmocr/olmocr_bench_fireworks.py deleted file mode 100644 index 0586801..0000000 --- a/benchmarks/olmocr/olmocr_bench_fireworks.py +++ /dev/null @@ -1,279 +0,0 @@ -""" -OlmOCR Benchmark for Fireworks-hosted models (default: Inkling). - -Mirrors olmocr_bench.py (interfaze) but routes pages through Fireworks' -OpenAI-compatible endpoint. - -Usage: - uv run -m benchmarks.olmocr.olmocr_bench_fireworks - uv run -m benchmarks.olmocr.olmocr_bench_fireworks --sample - uv run -m benchmarks.olmocr.olmocr_bench_fireworks --skip-generation - uv run -m benchmarks.olmocr.olmocr_bench_fireworks --generate-only - uv run -m benchmarks.olmocr.olmocr_bench_fireworks --model accounts/fireworks/models/inkling-small -""" - -import argparse -import asyncio -import json -import os -import re -import sys -from pathlib import Path - -from dotenv import load_dotenv -from huggingface_hub import hf_hub_download -from tqdm.asyncio import tqdm_asyncio - -load_dotenv() - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -SAMPLE_DATA_DIR = Path(__file__).resolve().parent / "bench" / "sample_data" -FULL_DATA_DIR = Path(__file__).resolve().parent / "bench" / "full_data" - -INKLING = "accounts/fireworks/models/inkling" -MODEL = INKLING -# Candidate name = the output subdirectory the olmOCR scorer reads. Derived from -# the model so inkling and inkling-small don't overwrite each other. -CANDIDATE_NAME = "inkling" -RATE_LIMIT = 25 -MAX_RETRIES = 3 - -# Cap on requests in flight. RATE_LIMIT bounds how many requests *start* per -# second but not how many are outstanding — all ~1400 pages launched at once -# queue server-side and eventually 429 after minutes of latency. Override with -# BENCH_MAX_IN_FLIGHT. -MAX_IN_FLIGHT = int(os.getenv("BENCH_MAX_IN_FLIGHT", "8")) - -HF_REPO = "allenai/olmOCR-bench" -SPLITS = [ - "arxiv_math", - "headers_footers", - "long_tiny_text", - "multi_column", - "old_scans", - "old_scans_math", - "table_tests", -] - -sys.path.insert(0, str(PROJECT_ROOT)) -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - - -def model_slug(model: str) -> str: - """accounts/fireworks/models/inkling-small -> inkling-small""" - return re.sub(r"[^a-z0-9]+", "-", model.rsplit("/", 1)[-1].lower()).strip("-") - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def download_full_dataset(): - data_dir = FULL_DATA_DIR - pdf_dir = data_dir / "pdfs" - all_pdfs = set() - for split in SPLITS: - jsonl_dest = data_dir / f"{split}.jsonl" - if jsonl_dest.exists(): - with open(jsonl_dest) as f: - tests = [json.loads(l) for l in f if l.strip()] - else: - print(f" Downloading {split}.jsonl...") - src = hf_hub_download( - HF_REPO, f"bench_data/{split}.jsonl", repo_type="dataset" - ) - with open(src) as f: - tests = [json.loads(l) for l in f if l.strip()] - data_dir.mkdir(parents=True, exist_ok=True) - with open(jsonl_dest, "w") as f: - for t in tests: - f.write(json.dumps(t) + "\n") - print(f" {split}: {len(tests)} tests") - for t in tests: - all_pdfs.add(t["pdf"]) - - print(f"\n Total unique PDFs to download: {len(all_pdfs)}") - downloaded = 0 - skipped = 0 - for pdf_rel in sorted(all_pdfs): - local_path = pdf_dir / pdf_rel - if local_path.exists(): - skipped += 1 - continue - local_path.parent.mkdir(parents=True, exist_ok=True) - try: - src = hf_hub_download( - HF_REPO, f"bench_data/pdfs/{pdf_rel}", repo_type="dataset" - ) - os.symlink(src, str(local_path)) - downloaded += 1 - except Exception as e: - print(f" Failed to download {pdf_rel}: {e}") - print(f" PDFs: {downloaded} downloaded, {skipped} already existed") - return data_dir - - -async def process_page(pdf_path, page_num, output_path, rate_limiter, sem): - async with sem: - return await _process_page(pdf_path, page_num, output_path, rate_limiter) - - -async def _process_page(pdf_path, page_num, output_path, rate_limiter): - from olmocr.bench.runners.run_fireworks import run_fireworks - - for attempt in range(MAX_RETRIES): - await rate_limiter.acquire() - try: - result = await asyncio.to_thread(run_fireworks, pdf_path, page_num, MODEL) - os.makedirs(os.path.dirname(output_path), exist_ok=True) - with open(output_path, "w", encoding="utf-8") as f: - f.write(result) - return True - except Exception as e: - if attempt < MAX_RETRIES - 1: - await asyncio.sleep(2**attempt) - else: - print( - f"Failed after {MAX_RETRIES} attempts: {pdf_path} page {page_num}: {e}" - ) - return False - - -async def generate_outputs(data_dir: Path): - pdf_folder = data_dir / "pdfs" - output_folder = data_dir / CANDIDATE_NAME - - pdf_pages = set() - for jsonl_file in data_dir.glob("*.jsonl"): - with open(jsonl_file) as f: - for line in f: - line = line.strip() - if not line: - continue - t = json.loads(line) - pdf_pages.add((t["pdf"], t["page"])) - - print(f"Found {len(pdf_pages)} unique (pdf, page) pairs to process") - - rate_limiter = RateLimiter(RATE_LIMIT) - sem = asyncio.Semaphore(MAX_IN_FLIGHT) - tasks = [] - for pdf_rel, page in sorted(pdf_pages): - pdf_path = str(pdf_folder / pdf_rel) - if not os.path.exists(pdf_path): - continue - base_name = os.path.splitext(os.path.basename(pdf_rel))[0] - parent_dir = os.path.dirname(pdf_rel) - md_filename = f"{base_name}_pg{page}_repeat1.md" - if parent_dir: - out_path = str(output_folder / parent_dir / md_filename) - else: - out_path = str(output_folder / md_filename) - if os.path.exists(out_path): - continue - tasks.append(process_page(pdf_path, page, out_path, rate_limiter, sem)) - - if not tasks: - print("All outputs already exist, skipping generation.") - return True - print(f"Processing {len(tasks)} pages...") - results = await tqdm_asyncio.gather( - *tasks, desc=f"Generating {CANDIDATE_NAME} outputs" - ) - num_success = sum(1 for r in results if r) - num_failed = len(results) - num_success - print(f"Done: {num_success} succeeded, {num_failed} failed") - return num_failed == 0 - - -class _Tee: - """olmOCR-bench prints its scores and saves nothing, so mirror stdout to a - log file — otherwise the numbers exist only in whatever terminal ran it and - scripts/report_scores.py has nothing to read.""" - - def __init__(self, *streams): - self.streams = streams - - def write(self, data): - for s in self.streams: - s.write(data) - s.flush() - - def flush(self): - for s in self.streams: - s.flush() - - -def run_evaluation(data_dir: Path): - from olmocr.bench.benchmark import main as bench_main - - sys.argv = [ - "benchmark", - "--dir", - str(data_dir), - "--candidate", - CANDIDATE_NAME, - "--force", - ] - log_path = PROJECT_ROOT / "logs" / f"olmocr_{CANDIDATE_NAME}.log" - log_path.parent.mkdir(parents=True, exist_ok=True) - real_stdout = sys.stdout - with open(log_path, "w", encoding="utf-8") as fh: - sys.stdout = _Tee(real_stdout, fh) - try: - bench_main() - finally: - sys.stdout = real_stdout - print(f"\nolmOCR scores also written to {log_path}") - - -async def main(): - global MODEL, CANDIDATE_NAME - parser = argparse.ArgumentParser( - description="Run OlmOCR benchmark against a Fireworks-hosted model" - ) - parser.add_argument( - "--model", default=INKLING, help="Fireworks model id (accounts/fireworks/...)" - ) - parser.add_argument("--sample", action="store_true") - parser.add_argument("--skip-generation", action="store_true") - parser.add_argument("--generate-only", action="store_true") - args = parser.parse_args() - - MODEL = args.model - CANDIDATE_NAME = model_slug(MODEL) - - if args.sample: - data_dir = SAMPLE_DATA_DIR - print("=== Using sample data ===") - else: - print("=== Downloading full olmOCR-bench dataset from HuggingFace ===") - data_dir = download_full_dataset() - - if not args.skip_generation: - print(f"\n=== Generating {CANDIDATE_NAME} outputs ===") - await generate_outputs(data_dir) - - if not args.generate_only: - print("\n=== Running OlmOCR Benchmark Evaluation ===") - run_evaluation(data_dir) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/benchmarks/olmocr/olmocr_bench_gemini.py b/benchmarks/olmocr/olmocr_bench_gemini.py deleted file mode 100644 index 415e77b..0000000 --- a/benchmarks/olmocr/olmocr_bench_gemini.py +++ /dev/null @@ -1,280 +0,0 @@ -""" -OlmOCR Benchmark for Gemini models (native google-genai, default 3.7 Flash). - -Mirrors olmocr_bench.py but routes pages through the native Gemini API, so -the methodology matches the repo's other --provider gemini runs (thinking_level -rather than OpenRouter's reasoning.effort). - -Usage: - uv run -m benchmarks.olmocr.olmocr_bench_gemini - uv run -m benchmarks.olmocr.olmocr_bench_gemini --sample - uv run -m benchmarks.olmocr.olmocr_bench_gemini --skip-generation - uv run -m benchmarks.olmocr.olmocr_bench_gemini --generate-only - uv run -m benchmarks.olmocr.olmocr_bench_gemini --model accounts/fireworks/models/inkling-small -""" - -import argparse -import asyncio -import json -import os -import re -import sys -from pathlib import Path - -from dotenv import load_dotenv -from huggingface_hub import hf_hub_download -from tqdm.asyncio import tqdm_asyncio - -load_dotenv() - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -SAMPLE_DATA_DIR = Path(__file__).resolve().parent / "bench" / "sample_data" -FULL_DATA_DIR = Path(__file__).resolve().parent / "bench" / "full_data" - -DEFAULT_MODEL = "gemini-3.7-flash" -MODEL = DEFAULT_MODEL -# Candidate name = the output subdirectory the olmOCR scorer reads. Derived from -# the model so inkling and inkling-small don't overwrite each other. -CANDIDATE_NAME = "gemini-3-7-flash" -RATE_LIMIT = 25 -MAX_RETRIES = 3 - -# Cap on requests in flight. RATE_LIMIT bounds how many requests *start* per -# second but not how many are outstanding — all ~1400 pages launched at once -# queue server-side and eventually get rate-limited. Override with -# BENCH_MAX_IN_FLIGHT. -MAX_IN_FLIGHT = int(os.getenv("BENCH_MAX_IN_FLIGHT", "8")) - -HF_REPO = "allenai/olmOCR-bench" -SPLITS = [ - "arxiv_math", - "headers_footers", - "long_tiny_text", - "multi_column", - "old_scans", - "old_scans_math", - "table_tests", -] - -sys.path.insert(0, str(PROJECT_ROOT)) -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - - -def model_slug(model: str) -> str: - """accounts/fireworks/models/inkling-small -> inkling-small""" - return re.sub(r"[^a-z0-9]+", "-", model.rsplit("/", 1)[-1].lower()).strip("-") - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def download_full_dataset(): - data_dir = FULL_DATA_DIR - pdf_dir = data_dir / "pdfs" - all_pdfs = set() - for split in SPLITS: - jsonl_dest = data_dir / f"{split}.jsonl" - if jsonl_dest.exists(): - with open(jsonl_dest) as f: - tests = [json.loads(l) for l in f if l.strip()] - else: - print(f" Downloading {split}.jsonl...") - src = hf_hub_download( - HF_REPO, f"bench_data/{split}.jsonl", repo_type="dataset" - ) - with open(src) as f: - tests = [json.loads(l) for l in f if l.strip()] - data_dir.mkdir(parents=True, exist_ok=True) - with open(jsonl_dest, "w") as f: - for t in tests: - f.write(json.dumps(t) + "\n") - print(f" {split}: {len(tests)} tests") - for t in tests: - all_pdfs.add(t["pdf"]) - - print(f"\n Total unique PDFs to download: {len(all_pdfs)}") - downloaded = 0 - skipped = 0 - for pdf_rel in sorted(all_pdfs): - local_path = pdf_dir / pdf_rel - if local_path.exists(): - skipped += 1 - continue - local_path.parent.mkdir(parents=True, exist_ok=True) - try: - src = hf_hub_download( - HF_REPO, f"bench_data/pdfs/{pdf_rel}", repo_type="dataset" - ) - os.symlink(src, str(local_path)) - downloaded += 1 - except Exception as e: - print(f" Failed to download {pdf_rel}: {e}") - print(f" PDFs: {downloaded} downloaded, {skipped} already existed") - return data_dir - - -async def process_page(pdf_path, page_num, output_path, rate_limiter, sem): - async with sem: - return await _process_page(pdf_path, page_num, output_path, rate_limiter) - - -async def _process_page(pdf_path, page_num, output_path, rate_limiter): - from olmocr.bench.runners.run_gemini import run_gemini - - for attempt in range(MAX_RETRIES): - await rate_limiter.acquire() - try: - result = await asyncio.to_thread(run_gemini, pdf_path, page_num, MODEL) - os.makedirs(os.path.dirname(output_path), exist_ok=True) - with open(output_path, "w", encoding="utf-8") as f: - f.write(result) - return True - except Exception as e: - if attempt < MAX_RETRIES - 1: - await asyncio.sleep(2**attempt) - else: - print( - f"Failed after {MAX_RETRIES} attempts: {pdf_path} page {page_num}: {e}" - ) - return False - - -async def generate_outputs(data_dir: Path): - pdf_folder = data_dir / "pdfs" - output_folder = data_dir / CANDIDATE_NAME - - pdf_pages = set() - for jsonl_file in data_dir.glob("*.jsonl"): - with open(jsonl_file) as f: - for line in f: - line = line.strip() - if not line: - continue - t = json.loads(line) - pdf_pages.add((t["pdf"], t["page"])) - - print(f"Found {len(pdf_pages)} unique (pdf, page) pairs to process") - - rate_limiter = RateLimiter(RATE_LIMIT) - sem = asyncio.Semaphore(MAX_IN_FLIGHT) - tasks = [] - for pdf_rel, page in sorted(pdf_pages): - pdf_path = str(pdf_folder / pdf_rel) - if not os.path.exists(pdf_path): - continue - base_name = os.path.splitext(os.path.basename(pdf_rel))[0] - parent_dir = os.path.dirname(pdf_rel) - md_filename = f"{base_name}_pg{page}_repeat1.md" - if parent_dir: - out_path = str(output_folder / parent_dir / md_filename) - else: - out_path = str(output_folder / md_filename) - if os.path.exists(out_path): - continue - tasks.append(process_page(pdf_path, page, out_path, rate_limiter, sem)) - - if not tasks: - print("All outputs already exist, skipping generation.") - return True - print(f"Processing {len(tasks)} pages...") - results = await tqdm_asyncio.gather( - *tasks, desc=f"Generating {CANDIDATE_NAME} outputs" - ) - num_success = sum(1 for r in results if r) - num_failed = len(results) - num_success - print(f"Done: {num_success} succeeded, {num_failed} failed") - return num_failed == 0 - - -class _Tee: - """olmOCR-bench prints its scores and saves nothing, so mirror stdout to a - log file — otherwise the numbers exist only in whatever terminal ran it and - scripts/report_scores.py has nothing to read.""" - - def __init__(self, *streams): - self.streams = streams - - def write(self, data): - for s in self.streams: - s.write(data) - s.flush() - - def flush(self): - for s in self.streams: - s.flush() - - -def run_evaluation(data_dir: Path): - from olmocr.bench.benchmark import main as bench_main - - sys.argv = [ - "benchmark", - "--dir", - str(data_dir), - "--candidate", - CANDIDATE_NAME, - "--force", - ] - log_path = PROJECT_ROOT / "logs" / f"olmocr_{CANDIDATE_NAME}.log" - log_path.parent.mkdir(parents=True, exist_ok=True) - real_stdout = sys.stdout - with open(log_path, "w", encoding="utf-8") as fh: - sys.stdout = _Tee(real_stdout, fh) - try: - bench_main() - finally: - sys.stdout = real_stdout - print(f"\nolmOCR scores also written to {log_path}") - - -async def main(): - global MODEL, CANDIDATE_NAME - parser = argparse.ArgumentParser( - description="Run OlmOCR benchmark against a Gemini model (native API)" - ) - parser.add_argument( - "--model", default=DEFAULT_MODEL, help="Gemini model id (e.g. gemini-3.7-flash)" - ) - parser.add_argument("--sample", action="store_true") - parser.add_argument("--skip-generation", action="store_true") - parser.add_argument("--generate-only", action="store_true") - args = parser.parse_args() - - MODEL = args.model - CANDIDATE_NAME = model_slug(MODEL) - - if args.sample: - data_dir = SAMPLE_DATA_DIR - print("=== Using sample data ===") - else: - print("=== Downloading full olmOCR-bench dataset from HuggingFace ===") - data_dir = download_full_dataset() - - if not args.skip_generation: - print(f"\n=== Generating {CANDIDATE_NAME} outputs ===") - await generate_outputs(data_dir) - - if not args.generate_only: - print("\n=== Running OlmOCR Benchmark Evaluation ===") - run_evaluation(data_dir) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/benchmarks/olmocr/olmocr_bench_gemini_pro_31.py b/benchmarks/olmocr/olmocr_bench_gemini_pro_31.py deleted file mode 100644 index aae674d..0000000 --- a/benchmarks/olmocr/olmocr_bench_gemini_pro_31.py +++ /dev/null @@ -1,202 +0,0 @@ -""" -OlmOCR Benchmark for Gemini 3.1 Pro Preview. - -Mirrors olmocr_bench.py (interfaze) but uses gemini-3.1-pro-preview. - -Usage: - uv run -m benchmarks.olmocr.olmocr_bench_gemini_pro_31 - uv run -m benchmarks.olmocr.olmocr_bench_gemini_pro_31 --sample - uv run -m benchmarks.olmocr.olmocr_bench_gemini_pro_31 --skip-generation - uv run -m benchmarks.olmocr.olmocr_bench_gemini_pro_31 --generate-only -""" - -import argparse -import asyncio -import json -import os -import sys -from pathlib import Path - -from dotenv import load_dotenv -from huggingface_hub import hf_hub_download -from tqdm.asyncio import tqdm_asyncio - -load_dotenv() - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -SAMPLE_DATA_DIR = Path(__file__).resolve().parent / "bench" / "sample_data" -FULL_DATA_DIR = Path(__file__).resolve().parent / "bench" / "full_data" -CANDIDATE_NAME = "gemini_pro_31" -RATE_LIMIT = 25 -MAX_RETRIES = 3 - -HF_REPO = "allenai/olmOCR-bench" -SPLITS = [ - "arxiv_math", - "headers_footers", - "long_tiny_text", - "multi_column", - "old_scans", - "old_scans_math", - "table_tests", -] - -sys.path.insert(0, str(PROJECT_ROOT)) -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def download_full_dataset(): - data_dir = FULL_DATA_DIR - pdf_dir = data_dir / "pdfs" - all_pdfs = set() - for split in SPLITS: - jsonl_dest = data_dir / f"{split}.jsonl" - if jsonl_dest.exists(): - with open(jsonl_dest) as f: - tests = [json.loads(l) for l in f if l.strip()] - else: - print(f" Downloading {split}.jsonl...") - src = hf_hub_download(HF_REPO, f"bench_data/{split}.jsonl", repo_type="dataset") - with open(src) as f: - tests = [json.loads(l) for l in f if l.strip()] - data_dir.mkdir(parents=True, exist_ok=True) - with open(jsonl_dest, "w") as f: - for t in tests: - f.write(json.dumps(t) + "\n") - print(f" {split}: {len(tests)} tests") - for t in tests: - all_pdfs.add(t["pdf"]) - - print(f"\n Total unique PDFs to download: {len(all_pdfs)}") - downloaded = 0 - skipped = 0 - for pdf_rel in sorted(all_pdfs): - local_path = pdf_dir / pdf_rel - if local_path.exists(): - skipped += 1 - continue - local_path.parent.mkdir(parents=True, exist_ok=True) - try: - src = hf_hub_download(HF_REPO, f"bench_data/pdfs/{pdf_rel}", repo_type="dataset") - os.symlink(src, str(local_path)) - downloaded += 1 - except Exception as e: - print(f" Failed to download {pdf_rel}: {e}") - print(f" PDFs: {downloaded} downloaded, {skipped} already existed") - return data_dir - - -async def process_page(pdf_path, page_num, output_path, rate_limiter): - from olmocr.bench.runners.run_gemini_pro_31 import run_gemini_pro_31 - - for attempt in range(MAX_RETRIES): - await rate_limiter.acquire() - try: - result = await asyncio.to_thread(run_gemini_pro_31, pdf_path, page_num) - os.makedirs(os.path.dirname(output_path), exist_ok=True) - with open(output_path, "w", encoding="utf-8") as f: - f.write(result) - return True - except Exception as e: - if attempt < MAX_RETRIES - 1: - await asyncio.sleep(2**attempt) - else: - print(f"Failed after {MAX_RETRIES} attempts: {pdf_path} page {page_num}: {e}") - return False - - -async def generate_outputs(data_dir: Path): - pdf_folder = data_dir / "pdfs" - output_folder = data_dir / CANDIDATE_NAME - - pdf_pages = set() - for jsonl_file in data_dir.glob("*.jsonl"): - with open(jsonl_file) as f: - for line in f: - line = line.strip() - if not line: - continue - t = json.loads(line) - pdf_pages.add((t["pdf"], t["page"])) - - print(f"Found {len(pdf_pages)} unique (pdf, page) pairs to process") - - rate_limiter = RateLimiter(RATE_LIMIT) - tasks = [] - for pdf_rel, page in sorted(pdf_pages): - pdf_path = str(pdf_folder / pdf_rel) - if not os.path.exists(pdf_path): - continue - base_name = os.path.splitext(os.path.basename(pdf_rel))[0] - parent_dir = os.path.dirname(pdf_rel) - md_filename = f"{base_name}_pg{page}_repeat1.md" - if parent_dir: - out_path = str(output_folder / parent_dir / md_filename) - else: - out_path = str(output_folder / md_filename) - if os.path.exists(out_path): - continue - tasks.append(process_page(pdf_path, page, out_path, rate_limiter)) - - if not tasks: - print("All outputs already exist, skipping generation.") - return True - print(f"Processing {len(tasks)} pages...") - results = await tqdm_asyncio.gather(*tasks, desc=f"Generating {CANDIDATE_NAME} outputs") - num_success = sum(1 for r in results if r) - num_failed = len(results) - num_success - print(f"Done: {num_success} succeeded, {num_failed} failed") - return num_failed == 0 - - -def run_evaluation(data_dir: Path): - from olmocr.bench.benchmark import main as bench_main - sys.argv = ["benchmark", "--dir", str(data_dir), "--candidate", CANDIDATE_NAME, "--force"] - bench_main() - - -async def main(): - parser = argparse.ArgumentParser(description=f"Run OlmOCR benchmark with {CANDIDATE_NAME}") - parser.add_argument("--sample", action="store_true") - parser.add_argument("--skip-generation", action="store_true") - parser.add_argument("--generate-only", action="store_true") - args = parser.parse_args() - - if args.sample: - data_dir = SAMPLE_DATA_DIR - print("=== Using sample data ===") - else: - print("=== Downloading full olmOCR-bench dataset from HuggingFace ===") - data_dir = download_full_dataset() - - if not args.skip_generation: - print(f"\n=== Generating {CANDIDATE_NAME} outputs ===") - await generate_outputs(data_dir) - - if not args.generate_only: - print("\n=== Running OlmOCR Benchmark Evaluation ===") - run_evaluation(data_dir) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/benchmarks/olmocr/olmocr_bench_grok.py b/benchmarks/olmocr/olmocr_bench_grok.py deleted file mode 100644 index 6a214da..0000000 --- a/benchmarks/olmocr/olmocr_bench_grok.py +++ /dev/null @@ -1,202 +0,0 @@ -""" -OlmOCR Benchmark for OpenAI Grok 4.3. - -Mirrors olmocr_bench.py (interfaze) but uses Grok 4.3 via OpenAI. - -Usage: - uv run -m benchmarks.olmocr.olmocr_bench_grok - uv run -m benchmarks.olmocr.olmocr_bench_grok --sample - uv run -m benchmarks.olmocr.olmocr_bench_grok --skip-generation - uv run -m benchmarks.olmocr.olmocr_bench_grok --generate-only -""" - -import argparse -import asyncio -import json -import os -import sys -from pathlib import Path - -from dotenv import load_dotenv -from huggingface_hub import hf_hub_download -from tqdm.asyncio import tqdm_asyncio - -load_dotenv() - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -SAMPLE_DATA_DIR = Path(__file__).resolve().parent / "bench" / "sample_data" -FULL_DATA_DIR = Path(__file__).resolve().parent / "bench" / "full_data" -CANDIDATE_NAME = "grok" -RATE_LIMIT = 25 -MAX_RETRIES = 3 - -HF_REPO = "allenai/olmOCR-bench" -SPLITS = [ - "arxiv_math", - "headers_footers", - "long_tiny_text", - "multi_column", - "old_scans", - "old_scans_math", - "table_tests", -] - -sys.path.insert(0, str(PROJECT_ROOT)) -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def download_full_dataset(): - data_dir = FULL_DATA_DIR - pdf_dir = data_dir / "pdfs" - all_pdfs = set() - for split in SPLITS: - jsonl_dest = data_dir / f"{split}.jsonl" - if jsonl_dest.exists(): - with open(jsonl_dest) as f: - tests = [json.loads(l) for l in f if l.strip()] - else: - print(f" Downloading {split}.jsonl...") - src = hf_hub_download(HF_REPO, f"bench_data/{split}.jsonl", repo_type="dataset") - with open(src) as f: - tests = [json.loads(l) for l in f if l.strip()] - data_dir.mkdir(parents=True, exist_ok=True) - with open(jsonl_dest, "w") as f: - for t in tests: - f.write(json.dumps(t) + "\n") - print(f" {split}: {len(tests)} tests") - for t in tests: - all_pdfs.add(t["pdf"]) - - print(f"\n Total unique PDFs to download: {len(all_pdfs)}") - downloaded = 0 - skipped = 0 - for pdf_rel in sorted(all_pdfs): - local_path = pdf_dir / pdf_rel - if local_path.exists(): - skipped += 1 - continue - local_path.parent.mkdir(parents=True, exist_ok=True) - try: - src = hf_hub_download(HF_REPO, f"bench_data/pdfs/{pdf_rel}", repo_type="dataset") - os.symlink(src, str(local_path)) - downloaded += 1 - except Exception as e: - print(f" Failed to download {pdf_rel}: {e}") - print(f" PDFs: {downloaded} downloaded, {skipped} already existed") - return data_dir - - -async def process_page(pdf_path, page_num, output_path, rate_limiter): - from olmocr.bench.runners.run_grok import run_grok - - for attempt in range(MAX_RETRIES): - await rate_limiter.acquire() - try: - result = await asyncio.to_thread(run_grok, pdf_path, page_num) - os.makedirs(os.path.dirname(output_path), exist_ok=True) - with open(output_path, "w", encoding="utf-8") as f: - f.write(result) - return True - except Exception as e: - if attempt < MAX_RETRIES - 1: - await asyncio.sleep(2**attempt) - else: - print(f"Failed after {MAX_RETRIES} attempts: {pdf_path} page {page_num}: {e}") - return False - - -async def generate_outputs(data_dir: Path): - pdf_folder = data_dir / "pdfs" - output_folder = data_dir / CANDIDATE_NAME - - pdf_pages = set() - for jsonl_file in data_dir.glob("*.jsonl"): - with open(jsonl_file) as f: - for line in f: - line = line.strip() - if not line: - continue - t = json.loads(line) - pdf_pages.add((t["pdf"], t["page"])) - - print(f"Found {len(pdf_pages)} unique (pdf, page) pairs to process") - - rate_limiter = RateLimiter(RATE_LIMIT) - tasks = [] - for pdf_rel, page in sorted(pdf_pages): - pdf_path = str(pdf_folder / pdf_rel) - if not os.path.exists(pdf_path): - continue - base_name = os.path.splitext(os.path.basename(pdf_rel))[0] - parent_dir = os.path.dirname(pdf_rel) - md_filename = f"{base_name}_pg{page}_repeat1.md" - if parent_dir: - out_path = str(output_folder / parent_dir / md_filename) - else: - out_path = str(output_folder / md_filename) - if os.path.exists(out_path): - continue - tasks.append(process_page(pdf_path, page, out_path, rate_limiter)) - - if not tasks: - print("All outputs already exist, skipping generation.") - return True - print(f"Processing {len(tasks)} pages...") - results = await tqdm_asyncio.gather(*tasks, desc=f"Generating {CANDIDATE_NAME} outputs") - num_success = sum(1 for r in results if r) - num_failed = len(results) - num_success - print(f"Done: {num_success} succeeded, {num_failed} failed") - return num_failed == 0 - - -def run_evaluation(data_dir: Path): - from olmocr.bench.benchmark import main as bench_main - sys.argv = ["benchmark", "--dir", str(data_dir), "--candidate", CANDIDATE_NAME, "--force"] - bench_main() - - -async def main(): - parser = argparse.ArgumentParser(description=f"Run OlmOCR benchmark with {CANDIDATE_NAME}") - parser.add_argument("--sample", action="store_true") - parser.add_argument("--skip-generation", action="store_true") - parser.add_argument("--generate-only", action="store_true") - args = parser.parse_args() - - if args.sample: - data_dir = SAMPLE_DATA_DIR - print("=== Using sample data ===") - else: - print("=== Downloading full olmOCR-bench dataset from HuggingFace ===") - data_dir = download_full_dataset() - - if not args.skip_generation: - print(f"\n=== Generating {CANDIDATE_NAME} outputs ===") - await generate_outputs(data_dir) - - if not args.generate_only: - print("\n=== Running OlmOCR Benchmark Evaluation ===") - run_evaluation(data_dir) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/benchmarks/olmocr/olmocr_bench_openai_mini.py b/benchmarks/olmocr/olmocr_bench_openai_mini.py deleted file mode 100644 index bfbdc00..0000000 --- a/benchmarks/olmocr/olmocr_bench_openai_mini.py +++ /dev/null @@ -1,202 +0,0 @@ -""" -OlmOCR Benchmark for OpenAI gpt-5.4-mini. - -Mirrors olmocr_bench.py (interfaze) but uses gpt-5.4-mini via OpenAI. - -Usage: - uv run -m benchmarks.olmocr.olmocr_bench_openai_mini - uv run -m benchmarks.olmocr.olmocr_bench_openai_mini --sample - uv run -m benchmarks.olmocr.olmocr_bench_openai_mini --skip-generation - uv run -m benchmarks.olmocr.olmocr_bench_openai_mini --generate-only -""" - -import argparse -import asyncio -import json -import os -import sys -from pathlib import Path - -from dotenv import load_dotenv -from huggingface_hub import hf_hub_download -from tqdm.asyncio import tqdm_asyncio - -load_dotenv() - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -SAMPLE_DATA_DIR = Path(__file__).resolve().parent / "bench" / "sample_data" -FULL_DATA_DIR = Path(__file__).resolve().parent / "bench" / "full_data" -CANDIDATE_NAME = "openai_mini" -RATE_LIMIT = 25 -MAX_RETRIES = 3 - -HF_REPO = "allenai/olmOCR-bench" -SPLITS = [ - "arxiv_math", - "headers_footers", - "long_tiny_text", - "multi_column", - "old_scans", - "old_scans_math", - "table_tests", -] - -sys.path.insert(0, str(PROJECT_ROOT)) -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -def download_full_dataset(): - data_dir = FULL_DATA_DIR - pdf_dir = data_dir / "pdfs" - all_pdfs = set() - for split in SPLITS: - jsonl_dest = data_dir / f"{split}.jsonl" - if jsonl_dest.exists(): - with open(jsonl_dest) as f: - tests = [json.loads(l) for l in f if l.strip()] - else: - print(f" Downloading {split}.jsonl...") - src = hf_hub_download(HF_REPO, f"bench_data/{split}.jsonl", repo_type="dataset") - with open(src) as f: - tests = [json.loads(l) for l in f if l.strip()] - data_dir.mkdir(parents=True, exist_ok=True) - with open(jsonl_dest, "w") as f: - for t in tests: - f.write(json.dumps(t) + "\n") - print(f" {split}: {len(tests)} tests") - for t in tests: - all_pdfs.add(t["pdf"]) - - print(f"\n Total unique PDFs to download: {len(all_pdfs)}") - downloaded = 0 - skipped = 0 - for pdf_rel in sorted(all_pdfs): - local_path = pdf_dir / pdf_rel - if local_path.exists(): - skipped += 1 - continue - local_path.parent.mkdir(parents=True, exist_ok=True) - try: - src = hf_hub_download(HF_REPO, f"bench_data/pdfs/{pdf_rel}", repo_type="dataset") - os.symlink(src, str(local_path)) - downloaded += 1 - except Exception as e: - print(f" Failed to download {pdf_rel}: {e}") - print(f" PDFs: {downloaded} downloaded, {skipped} already existed") - return data_dir - - -async def process_page(pdf_path, page_num, output_path, rate_limiter): - from olmocr.bench.runners.run_openai_mini import run_openai_mini - - for attempt in range(MAX_RETRIES): - await rate_limiter.acquire() - try: - result = await asyncio.to_thread(run_openai_mini, pdf_path, page_num) - os.makedirs(os.path.dirname(output_path), exist_ok=True) - with open(output_path, "w", encoding="utf-8") as f: - f.write(result) - return True - except Exception as e: - if attempt < MAX_RETRIES - 1: - await asyncio.sleep(2**attempt) - else: - print(f"Failed after {MAX_RETRIES} attempts: {pdf_path} page {page_num}: {e}") - return False - - -async def generate_outputs(data_dir: Path): - pdf_folder = data_dir / "pdfs" - output_folder = data_dir / CANDIDATE_NAME - - pdf_pages = set() - for jsonl_file in data_dir.glob("*.jsonl"): - with open(jsonl_file) as f: - for line in f: - line = line.strip() - if not line: - continue - t = json.loads(line) - pdf_pages.add((t["pdf"], t["page"])) - - print(f"Found {len(pdf_pages)} unique (pdf, page) pairs to process") - - rate_limiter = RateLimiter(RATE_LIMIT) - tasks = [] - for pdf_rel, page in sorted(pdf_pages): - pdf_path = str(pdf_folder / pdf_rel) - if not os.path.exists(pdf_path): - continue - base_name = os.path.splitext(os.path.basename(pdf_rel))[0] - parent_dir = os.path.dirname(pdf_rel) - md_filename = f"{base_name}_pg{page}_repeat1.md" - if parent_dir: - out_path = str(output_folder / parent_dir / md_filename) - else: - out_path = str(output_folder / md_filename) - if os.path.exists(out_path): - continue - tasks.append(process_page(pdf_path, page, out_path, rate_limiter)) - - if not tasks: - print("All outputs already exist, skipping generation.") - return True - print(f"Processing {len(tasks)} pages...") - results = await tqdm_asyncio.gather(*tasks, desc=f"Generating {CANDIDATE_NAME} outputs") - num_success = sum(1 for r in results if r) - num_failed = len(results) - num_success - print(f"Done: {num_success} succeeded, {num_failed} failed") - return num_failed == 0 - - -def run_evaluation(data_dir: Path): - from olmocr.bench.benchmark import main as bench_main - sys.argv = ["benchmark", "--dir", str(data_dir), "--candidate", CANDIDATE_NAME, "--force"] - bench_main() - - -async def main(): - parser = argparse.ArgumentParser(description=f"Run OlmOCR benchmark with {CANDIDATE_NAME}") - parser.add_argument("--sample", action="store_true") - parser.add_argument("--skip-generation", action="store_true") - parser.add_argument("--generate-only", action="store_true") - args = parser.parse_args() - - if args.sample: - data_dir = SAMPLE_DATA_DIR - print("=== Using sample data ===") - else: - print("=== Downloading full olmOCR-bench dataset from HuggingFace ===") - data_dir = download_full_dataset() - - if not args.skip_generation: - print(f"\n=== Generating {CANDIDATE_NAME} outputs ===") - await generate_outputs(data_dir) - - if not args.generate_only: - print("\n=== Running OlmOCR Benchmark Evaluation ===") - run_evaluation(data_dir) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/scripts/run_fireworks_suite.sh b/scripts/run_fireworks_suite.sh index 0884ba1..fedce81 100755 --- a/scripts/run_fireworks_suite.sh +++ b/scripts/run_fireworks_suite.sh @@ -93,7 +93,11 @@ else echo " [skip] refcoco: set TARGET= (python -m bench_core list-targets)" fi -run_step olmocr uv run -m benchmarks.olmocr.olmocr_bench_fireworks --model "$MODEL" +if [ -n "${TARGET:-}" ]; then + run_step olmocr uv run python -m bench_core run --benchmark olmocr --target "$TARGET" +else + echo " [skip] olmocr: set TARGET= (python -m bench_core list-targets)" +fi if [ -n "${TARGET:-}" ]; then run_step mmmlu uv run python -m bench_core run --benchmark mmmlu --target "$TARGET" \ diff --git a/tests/test_olmocr_bench.py b/tests/test_olmocr_bench.py new file mode 100644 index 0000000..2b57acb --- /dev/null +++ b/tests/test_olmocr_bench.py @@ -0,0 +1,39 @@ +"""olmOCR: unit tests for the null-normalization and log parsing. + +olmOCR has no offline score-replay (its scorer needs the full dataset + prints to +a log, no metrics.json). Parity is by-construction: identical PROMPT, PNG render, +and null handling. Here we pin those and the log parser against the archived log. +""" + +import pathlib +from types import SimpleNamespace as NS + +from benchmarks.olmocr import harness as bench + +LOGS = pathlib.Path(__file__).resolve().parent.parent / "logs" + + +def test_parse_null_becomes_empty(): + assert bench.parse(NS(text="null"), None) == "" + assert bench.parse(NS(text=" N/A "), None) == "" + assert bench.parse(NS(text="Real content."), None) == "Real content." + + +def test_prompt_is_stable(): + # the scorer's numbers depend on this prompt; guard against silent drift + assert "Turn tables into markdown format." in bench.PROMPT + assert "Do not hallucinate." in bench.PROMPT + + +def test_parse_scores_from_archived_log(): + log = LOGS / "olmocr_gemini-3-7-flash.log" + if not log.exists(): + import pytest + + pytest.skip("archived olmOCR log not present") + scores = bench._parse_scores(log.read_text(errors="ignore")) + assert scores["overall"] == 77.3 + assert scores["splits"]["arxiv_math"] == 86.7 + assert scores["splits"]["old_scans"] == 44.9 + # matches report_scores.load_olmocr: the first `baseline:` line (summary block) + assert scores["splits"]["baseline"] == 93.4 From b4657d37277eb607926a5b595113483424be91ff Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 09:12:09 +0530 Subject: [PATCH 14/38] feat(bench_core): migrate Spider2-Lite onto the abstraction; retire the fork MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit benchmarks/spider2_lite/bench.py routes text-to-SQL through bench_core and reuses the SQL extraction, in-memory SQLite execution (120s timeout guard), and result-set comparison. Retires spider2_lite.py; fetch_data.py (data downloader) kept. Offline unit tests cover extract_sql (incl. the unterminated-fence fallback) and the result comparator. Full score parity re-executes SQL against the ~4GB gitignored data/, so it is gated behind data/ + BENCH_SPIDER2_PARITY=1; run locally it reproduces inkling within execution noise (65 vs archived 64 — a heavy query flips on the 120s timeout boundary, inherent to Spider2, not a port bug). 158 tests pass, 1 gated skip. --- README.md | 12 +- bench_core/cli.py | 1 + benchmarks/spider2_lite/bench.py | 305 ++++++++++ benchmarks/spider2_lite/spider2_lite.py | 778 ------------------------ scripts/run_fireworks_suite.sh | 7 +- tests/test_spider2_bench.py | 72 +++ 6 files changed, 388 insertions(+), 787 deletions(-) create mode 100644 benchmarks/spider2_lite/bench.py delete mode 100644 benchmarks/spider2_lite/spider2_lite.py create mode 100644 tests/test_spider2_bench.py diff --git a/README.md b/README.md index a638dae..4ced58d 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ uv run python -m bench_core run --target inkling --benchmark mmmu_pro --variant uv run python -m bench_core run --target inkling --benchmark mmmu_pro --variant vision uv run python -m bench_core run --target inkling --benchmark refcoco --variant val uv run python -m bench_core run --target inkling --benchmark asr -uv run -m benchmarks.spider2_lite.spider2_lite --provider fireworks +uv run python -m bench_core run --target inkling --benchmark spider2 ``` Two caveats when reading Fireworks numbers: @@ -258,11 +258,9 @@ Text-to-SQL with execution-accuracy scoring against per-example SQLite databases ```bash # One-time setup: clone Spider2 + download SQLite databases -uv run -m benchmarks.spider2_lite.fetch_data +uv run -m benchmarks.spider2_lite.fetch_data # downloads the ~4GB data/ (required) -# Run -uv run -m benchmarks.spider2_lite.spider2_lite -uv run -m benchmarks.spider2_lite.spider2_lite --provider fireworks -uv run -m benchmarks.spider2_lite.spider2_lite --predict-only -uv run -m benchmarks.spider2_lite.spider2_lite --evaluate-only +# Run any target (execution accuracy over the 135 local SQLite instances) +uv run python -m bench_core run --target gpt-5.5 --benchmark spider2 +uv run python -m bench_core run --target inkling --benchmark spider2 ``` diff --git a/bench_core/cli.py b/bench_core/cli.py index dd142a3..6486ecc 100644 --- a/bench_core/cli.py +++ b/bench_core/cli.py @@ -25,6 +25,7 @@ "refcoco": "benchmarks.obj_detection.bench", "ocrbench_v2": "benchmarks.ocrbench_v2.bench", "olmocr": "benchmarks.olmocr.harness", + "spider2": "benchmarks.spider2_lite.bench", } diff --git a/benchmarks/spider2_lite/bench.py b/benchmarks/spider2_lite/bench.py new file mode 100644 index 0000000..b704d9d --- /dev/null +++ b/benchmarks/spider2_lite/bench.py @@ -0,0 +1,305 @@ +"""Spider2-Lite (SQLite subset): text-to-SQL; execution accuracy over 135 local +instances. Scoring re-executes predicted SQL against the local .sqlite DBs and +compares result sets to gold — it needs the (gitignored) data/ directory. +""" + +from __future__ import annotations + +import csv +import math +import os +import re +import sqlite3 +import time +from pathlib import Path + +from bench_core.request import Message, ReasoningSpec, Request, TextPart + +NAME = "spider2_lite" +ID_KEY = "instance_id" +PRIMARY_METRIC = "accuracy_of_local_135" +DEFAULTS = {"reasoning": "off", "rate_limit": 8, "max_in_flight": 8} + +_DATA_DIR = Path(__file__).resolve().parent / "data" +_SPIDER2_LITE = _DATA_DIR / "Spider2" / "spider2-lite" +_SQLITE_DB_DIR = _SPIDER2_LITE / "resource" / "databases" +_SCHEMA_DIR = _SQLITE_DB_DIR / "sqlite" +_DOCUMENTS_DIR = _SPIDER2_LITE / "resource" / "documents" +_GOLD_DIR = _SPIDER2_LITE / "evaluation_suite" / "gold" +_EVAL_STANDARD = _GOLD_DIR / "spider2lite_eval.jsonl" +_GOLD_EXEC_DIR = _GOLD_DIR / "exec_result" +_ALL_EXAMPLES = _SPIDER2_LITE / "spider2-lite.jsonl" + +QUERY_TIMEOUT_S = 120.0 +MAX_DDL_CHARS = 80_000 +MAX_EK_CHARS = 40_000 + +PROMPT_TEMPLATE = """You are an expert SQLite SQL developer. Write a SQL query that answers the user's question against the given database. Target dialect: SQLite. + +### Database Schema +{schema} +{external_knowledge_section} +### Question +{question} + +Return ONLY the final SQL query, wrapped in a fenced code block like: +```sql +SELECT ... +``` +Do not include any explanation before or after the code block.""" + +_SQL_FENCE = re.compile(r"```sql\s*\n(.*?)```", re.IGNORECASE | re.DOTALL) +_ANY_FENCE = re.compile(r"```\s*\n?(.*?)```", re.DOTALL) +# Unclosed fence: an opener with no terminator (some models never close the block). +_OPEN_FENCE = re.compile(r"```(?:sql)?[ \t]*\r?\n(.*)\Z", re.IGNORECASE | re.DOTALL) + + +def extract_sql(text: str) -> str: + if not text: + return "" + for pat in (_SQL_FENCE, _ANY_FENCE, _OPEN_FENCE): + m = pat.search(text) + if m: + return m.group(1).strip() + return text.strip() + + +# --- prompt assembly (needs data/) --- +def load_schema(db_name: str) -> str: + ddl_path = _SCHEMA_DIR / db_name / "DDL.csv" + if not ddl_path.exists(): + return f"-- schema file missing: {ddl_path}" + parts = [] + with open(ddl_path, encoding="utf-8") as f: + for row in csv.DictReader(f): + ddl = (row.get("DDL") or "").strip() + if ddl: + parts.append(ddl.rstrip(";") + ";") + schema = "\n\n".join(parts) + if len(schema) > MAX_DDL_CHARS: + schema = schema[:MAX_DDL_CHARS] + "\n-- [schema truncated]" + return schema + + +def load_external_knowledge(filename: str | None) -> str | None: + if not filename: + return None + path = _DOCUMENTS_DIR / filename + if not path.exists(): + return None + text = path.read_text(encoding="utf-8", errors="ignore") + if len(text) > MAX_EK_CHARS: + text = text[:MAX_EK_CHARS] + "\n... [truncated]" + return text + + +def build_prompt(example: dict) -> str: + ek = load_external_knowledge(example.get("external_knowledge")) + ek_section = f"\n### External Knowledge\n{ek}\n" if ek else "\n" + return PROMPT_TEMPLATE.format( + schema=load_schema(example["db"]), + external_knowledge_section=ek_section, + question=example["question"], + ) + + +def load_samples(sample_size: int | None = None) -> list[dict]: + if not _ALL_EXAMPLES.exists(): + raise FileNotFoundError( + f"Missing {_ALL_EXAMPLES}. The Spider2 data/ dir (gitignored, ~4GB) must " + "be present. Run: uv run -m benchmarks.spider2_lite.fetch_data" + ) + import json + + rows = [ + json.loads(line) + for line in _ALL_EXAMPLES.read_text().splitlines() + if line.strip() + ] + local = [r for r in rows if r["instance_id"].startswith("local")] + samples = [] + for ex in local: + samples.append( + { + "instance_id": ex["instance_id"], + "db": ex["db"], + "question": ex["question"], + "external_knowledge": ex.get("external_knowledge"), + "prompt": build_prompt(ex), + } + ) + return samples[:sample_size] if sample_size else samples + + +def build_request(sample: dict, mode: str) -> Request: + return Request( + [Message("user", [TextPart(sample["prompt"])])], + reasoning=ReasoningSpec(mode), + temperature=0.0, + ) + + +def parse(response, sample) -> str: + return extract_sql(response.text or "") + + +# --- execution + comparison (verbatim port of the official evaluator's quirks) --- +def execute_sqlite(db_path: Path, sql: str): + try: + disk = sqlite3.connect(str(db_path)) + mem = sqlite3.connect(":memory:") + try: + import pandas as pd + + disk.backup(mem) + deadline = time.monotonic() + QUERY_TIMEOUT_S + # Non-zero from the handler aborts the query (unbounded joins otherwise + # pin CPU forever and hang the whole scoring pass). + mem.set_progress_handler( + lambda: 1 if time.monotonic() > deadline else 0, 10_000 + ) + df = pd.read_sql_query(sql, mem) + mem.set_progress_handler(None, 0) + return True, df + finally: + mem.close() + disk.close() + except Exception as e: # noqa: BLE001 — any failure = failed execution (score 0) + return False, f"{type(e).__name__}: {e}" + + +def _normalize(v): + import pandas as pd + + return 0 if pd.isna(v) else v + + +def _sort_key(x): + return (x is None, str(x), isinstance(x, (int, float))) + + +def _vectors_match(v1, v2, ignore_order: bool, tol: float = 1e-2) -> bool: + import pandas as pd + + v1 = [_normalize(x) for x in v1] + v2 = [_normalize(x) for x in v2] + if ignore_order: + v1 = sorted(v1, key=_sort_key) + v2 = sorted(v2, key=_sort_key) + if len(v1) != len(v2): + return False + for a, b in zip(v1, v2): + if pd.isna(a) and pd.isna(b): + continue + if isinstance(a, (int, float)) and isinstance(b, (int, float)): + if not math.isclose(float(a), float(b), abs_tol=tol): + return False + elif a != b: + return False + return True + + +def compare_table(pred, gold, condition_cols, ignore_order: bool) -> int: + if condition_cols: + if not isinstance(condition_cols, (list, tuple)): + condition_cols = [condition_cols] + gold = gold.iloc[:, condition_cols] + t_gold = gold.transpose().values.tolist() + t_pred = pred.transpose().values.tolist() + for gv in t_gold: + if not any(_vectors_match(gv, pv, ignore_order) for pv in t_pred): + return 0 + return 1 + + +def compare_multi(pred, golds, multi_condition_cols, ignore_order: bool) -> int: + if not golds: + return 0 + if multi_condition_cols in (None, [], [[]], [None]): + multi_condition_cols = [[] for _ in golds] + elif len(golds) > 1 and not all(isinstance(s, list) for s in multi_condition_cols): + multi_condition_cols = [multi_condition_cols for _ in golds] + for gold, cc in zip(golds, multi_condition_cols): + if compare_table(pred, gold, cc, ignore_order): + return 1 + return 0 + + +def resolve_gold_paths(instance_id: str): + base = _GOLD_EXEC_DIR / f"{instance_id}.csv" + if base.exists(): + return [base], True + pattern = re.compile(rf"^{re.escape(instance_id)}(_[a-z])?\.csv$") + matches = sorted( + _GOLD_EXEC_DIR / name + for name in os.listdir(_GOLD_EXEC_DIR) + if pattern.match(name) + ) + return matches, False + + +def load_eval_standard() -> dict: + import json + + out = {} + with open(_EVAL_STANDARD, encoding="utf-8") as f: + for line in f: + if line.strip(): + rec = json.loads(line) + out[rec["instance_id"]] = rec + return out + + +def evaluate_record(instance_id: str, db: str, pred_sql: str, eval_std: dict) -> dict: + import pandas as pd + + db_path = _SQLITE_DB_DIR / f"{db}.sqlite" + if not db_path.exists(): + return { + "instance_id": instance_id, + "score": 0, + "error": f"missing sqlite db: {db_path}", + } + if not (pred_sql or "").strip(): + return {"instance_id": instance_id, "score": 0, "error": "empty pred_sql"} + ok, result = execute_sqlite(db_path, pred_sql) + if not ok: + return {"instance_id": instance_id, "score": 0, "error": f"sql error: {result}"} + gold_paths, is_single = resolve_gold_paths(instance_id) + if not gold_paths: + return {"instance_id": instance_id, "score": 0, "error": "no gold file"} + std = eval_std.get(instance_id, {}) + cc, ignore_order = std.get("condition_cols"), std.get("ignore_order", False) + try: + if is_single: + score = compare_table(result, pd.read_csv(gold_paths[0]), cc, ignore_order) + else: + score = compare_multi( + result, [pd.read_csv(p) for p in gold_paths], cc, ignore_order + ) + except Exception as e: # noqa: BLE001 + return {"instance_id": instance_id, "score": 0, "error": f"compare: {e}"} + return {"instance_id": instance_id, "score": score, "error": None} + + +def score(records: list[dict], samples: list[dict]) -> dict: + by_id = {s["instance_id"]: s for s in samples} + latest = {r["instance_id"]: r for r in records if r["instance_id"] in by_id} + eval_std = load_eval_standard() + per_example, correct = [], 0 + for iid, r in latest.items(): + res = evaluate_record( + iid, by_id[iid]["db"], r.get("prediction") or "", eval_std + ) + per_example.append(res) + correct += res["score"] + total = len(latest) + total_local = len(samples) + return { + "accuracy_of_local_135": correct / total_local if total_local else 0.0, + "accuracy_evaluated": correct / total if total else 0.0, + "correct": correct, + "total_evaluated": total, + "total_local_subset": total_local, + "per_example": per_example, + } diff --git a/benchmarks/spider2_lite/spider2_lite.py b/benchmarks/spider2_lite/spider2_lite.py deleted file mode 100644 index 74416c7..0000000 --- a/benchmarks/spider2_lite/spider2_lite.py +++ /dev/null @@ -1,778 +0,0 @@ -""" -Spider 2.0-Lite — SQLite subset — text-to-SQL benchmark for Interfaze. - -This runs the 135 `local*` examples from Spider 2.0-Lite (the SQLite-backed -slice). Scoring is execution accuracy: predicted SQL is executed against the -per-example `.sqlite` file and the result DataFrame is compared to the gold -exec_result CSV(s). Comparison logic is ported from the official -`evaluation_suite/evaluate.py` so scores are directly consistent with the -reference implementation. - -Important caveat: this is ~25% of the full Spider 2.0-Lite benchmark (135/547). -The BigQuery and Snowflake subsets are excluded because they need external -warehouse access. A result here is reportable as "Spider 2.0-Lite (SQLite -subset, N=135)" — NOT as the headline Spider 2.0-Lite score. - -Setup (one-time): - uv run -m benchmarks.spider2_lite.fetch_data - -Usage: - # Full run (predict + evaluate) - uv run -m benchmarks.spider2_lite.spider2_lite - - # Prediction only - uv run -m benchmarks.spider2_lite.spider2_lite --predict-only - - # Evaluation only - uv run -m benchmarks.spider2_lite.spider2_lite --evaluate-only - - # Smoke test - uv run -m benchmarks.spider2_lite.spider2_lite --limit 1 - -Checkpointing: each successful prediction is appended to -`results/spider2_lite_local_responses.jsonl` AND written as -`results/spider2_lite_local_sql/.sql`. Reruns only query -examples still missing from the JSONL. -""" - -from __future__ import annotations - -import argparse -import asyncio -import csv -import json -import math -import os -import re -import sys -import sqlite3 -import time -import traceback -from pathlib import Path - -import pandas as pd -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -from src.commons import invoke_interfaze # noqa: E402 - -# Paths ----------------------------------------------------------------------- - -DATA_DIR = Path(__file__).resolve().parent / "data" -SPIDER2_LITE = DATA_DIR / "Spider2" / "spider2-lite" -SQLITE_DB_DIR = SPIDER2_LITE / "resource" / "databases" -SCHEMA_DIR = SPIDER2_LITE / "resource" / "databases" / "sqlite" -DOCUMENTS_DIR = SPIDER2_LITE / "resource" / "documents" -GOLD_DIR = SPIDER2_LITE / "evaluation_suite" / "gold" -EVAL_STANDARD = GOLD_DIR / "spider2lite_eval.jsonl" -GOLD_EXEC_DIR = GOLD_DIR / "exec_result" -ALL_EXAMPLES = SPIDER2_LITE / "spider2-lite.jsonl" - -RESULTS_DIR = PROJECT_ROOT / "results" -TAG = "spider2_lite_local" -RESPONSES_PATH = RESULTS_DIR / f"{TAG}_responses.jsonl" -PRED_SQL_DIR = RESULTS_DIR / f"{TAG}_sql" -METRICS_PATH = RESULTS_DIR / f"{TAG}_metrics.json" - -# Config ---------------------------------------------------------------------- - -# Provider/model, set from the CLI. interfaze is the default so existing -# invocations and output paths are unchanged. -PROVIDER = "interfaze" -MODEL = "interfaze-beta" - -# Per-query wall-clock cap during scoring — see execute_sqlite. -QUERY_TIMEOUT_S = 120.0 - -# reasoning.effort sent to OpenRouter-hosted models. Override per run. -OPENROUTER_EFFORT = os.getenv("BENCH_OPENROUTER_EFFORT", "none") - -_GEMINI_CLIENT = None -# Gemini thinking_level: "auto" = the model's floor (3.7 rejects minimal, so -# "low"), or an explicit minimal|low|medium|high. Set from the CLI. -GEMINI_LEVEL = "auto" - -REASONING_EFFORT = None -TEMPERATURE = 0.0 -RATE_LIMIT = 8 -MAX_RETRIES = 3 -# Truncate huge DDLs / external-knowledge docs so a single monster schema -# doesn't blow the context. 80k chars leaves plenty of room for reasoning -# output on a 200k-token context model. -MAX_DDL_CHARS = 80_000 -MAX_EK_CHARS = 40_000 - -PROMPT_TEMPLATE = """You are an expert SQLite SQL developer. Write a SQL query that answers the user's question against the given database. Target dialect: SQLite. - -### Database Schema -{schema} -{external_knowledge_section} -### Question -{question} - -Return ONLY the final SQL query, wrapped in a fenced code block like: -```sql -SELECT ... -``` -Do not include any explanation before or after the code block.""" - - -# ----------------------------------------------------------------------------- -# Utilities shared with other benches -# ----------------------------------------------------------------------------- - - -class RateLimiter: - def __init__(self, rate: int): - self.rate = rate - self.tokens = rate - self.last_refill = 0.0 - self._lock = asyncio.Lock() - - async def acquire(self): - while True: - async with self._lock: - now = asyncio.get_running_loop().time() - elapsed = now - self.last_refill - self.tokens = min(self.rate, self.tokens + elapsed * self.rate) - self.last_refill = now - if self.tokens >= 1: - self.tokens -= 1 - return - await asyncio.sleep(1 / self.rate) - - -class JsonlWriter: - def __init__(self, path: Path): - self.path = path - self.path.parent.mkdir(parents=True, exist_ok=True) - self._lock = asyncio.Lock() - - async def append(self, record: dict): - line = json.dumps(record, ensure_ascii=False) - async with self._lock: - with open(self.path, "a", encoding="utf-8") as f: - f.write(line + "\n") - f.flush() - os.fsync(f.fileno()) - - -# ----------------------------------------------------------------------------- -# Dataset loading -# ----------------------------------------------------------------------------- - - -def load_local_examples() -> list[dict]: - if not ALL_EXAMPLES.exists(): - raise FileNotFoundError( - f"Missing {ALL_EXAMPLES}. Run: uv run -m benchmarks.spider2_lite.fetch_data" - ) - with open(ALL_EXAMPLES, encoding="utf-8") as f: - rows = [json.loads(line) for line in f if line.strip()] - return [r for r in rows if r["instance_id"].startswith("local")] - - -def load_schema(db_name: str) -> str: - """Concatenate every table's DDL from resource/databases/sqlite//DDL.csv. - DDL.csv is a 2-column file: `table_name,DDL`. Some DDL strings span lines, - so use csv.DictReader rather than hand-parsing.""" - ddl_path = SCHEMA_DIR / db_name / "DDL.csv" - if not ddl_path.exists(): - return f"-- schema file missing: {ddl_path}" - parts: list[str] = [] - with open(ddl_path, encoding="utf-8") as f: - for row in csv.DictReader(f): - ddl = (row.get("DDL") or "").strip() - if ddl: - parts.append(ddl.rstrip(";") + ";") - schema = "\n\n".join(parts) - if len(schema) > MAX_DDL_CHARS: - schema = schema[:MAX_DDL_CHARS] + "\n-- [schema truncated]" - return schema - - -def load_external_knowledge(filename: str | None) -> str | None: - if not filename: - return None - path = DOCUMENTS_DIR / filename - if not path.exists(): - return None - text = path.read_text(encoding="utf-8", errors="ignore") - if len(text) > MAX_EK_CHARS: - text = text[:MAX_EK_CHARS] + "\n... [truncated]" - return text - - -def build_prompt(example: dict) -> str: - schema = load_schema(example["db"]) - ek = load_external_knowledge(example.get("external_knowledge")) - ek_section = f"\n### External Knowledge\n{ek}\n" if ek else "\n" - return PROMPT_TEMPLATE.format( - schema=schema, - external_knowledge_section=ek_section, - question=example["question"], - ) - - -# ----------------------------------------------------------------------------- -# Response parsing -# ----------------------------------------------------------------------------- - -_SQL_FENCE = re.compile(r"```sql\s*\n(.*?)```", re.IGNORECASE | re.DOTALL) -_ANY_FENCE = re.compile(r"```\s*\n?(.*?)```", re.DOTALL) -# Unclosed fence: an opener with no terminator. Some models (deepseek-v4-flash -# on 57/135 Spider2 examples) emit complete SQL ending in ';' but never close -# the block. Without this, the whole response — opening fence included — was -# handed to sqlite as SQL and failed with `unrecognized token: "```sql`, -# scoring a valid query as wrong. -_OPEN_FENCE = re.compile(r"```(?:sql)?[ \t]*\r?\n(.*)\Z", re.IGNORECASE | re.DOTALL) - - -def extract_sql(text: str) -> str: - """Mirrors the behavior of the official evaluate.py: prefer an ```sql - fenced block, otherwise treat the whole response as SQL. Also tolerate - a plain ``` fence without the `sql` tag.""" - if not text: - return "" - m = _SQL_FENCE.search(text) - if m: - return m.group(1).strip() - m = _ANY_FENCE.search(text) - if m: - return m.group(1).strip() - m = _OPEN_FENCE.search(text) - if m: - return m.group(1).strip() - return text.strip() - - -# ----------------------------------------------------------------------------- -# Checkpoint helpers -# ----------------------------------------------------------------------------- - - -def load_completed_ids(path: Path) -> set[str]: - if not path.exists(): - return set() - done: set[str] = set() - with open(path, encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - rec = json.loads(line) - except json.JSONDecodeError: - continue - if rec.get("pred_sql"): - done.add(str(rec["instance_id"])) - return done - - -def load_records(path: Path) -> list[dict]: - if not path.exists(): - return [] - by_id: dict[str, dict] = {} - with open(path, encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - rec = json.loads(line) - except json.JSONDecodeError: - continue - by_id[str(rec["instance_id"])] = rec - return list(by_id.values()) - - -# ----------------------------------------------------------------------------- -# Prediction -# ----------------------------------------------------------------------------- - - -def invoke_model(messages: list[dict]): - """Dispatch to the configured provider — both endpoints are OpenAI-shaped. - - Inkling always thinks: reasoning_effort="none" is its floor, so a Fireworks - run is not directly comparable to the interfaze reasoning-off run.""" - if PROVIDER == "gemini": - # google-genai returns resp.text, but every caller here expects an - # OpenAI-shaped object, so wrap it rather than special-casing downstream. - from google import genai - from google.genai import types - - key = ( - os.getenv("GEMINI_API_KEY") - or os.getenv("GEMINI_KEY") - or os.getenv("GOOGLE_API_KEY") - ) - if not key: - raise ValueError("GEMINI_API_KEY missing from .env") - if GEMINI_LEVEL != "auto": - level = GEMINI_LEVEL - elif any(v in MODEL.lower() for v in ("3.7", "3.8", "3.9", "4.", "pro")): - level = "low" - else: - level = "minimal" - # Cache the client: constructing one per call gets it garbage-collected - # mid-flight ("Cannot send a request, as the client has been closed"). - global _GEMINI_CLIENT - if _GEMINI_CLIENT is None: - _GEMINI_CLIENT = genai.Client(api_key=key) - resp = _GEMINI_CLIENT.models.generate_content( - model=MODEL, - contents=[messages[0]["content"]], - config=types.GenerateContentConfig( - temperature=TEMPERATURE, - thinking_config=types.ThinkingConfig(thinking_level=level), - ), - ) - - class _Shim: - def __init__(self, text, rid): - self.choices = [ - type("C", (), {"message": type("M", (), {"content": text})()})() - ] - self.id = rid - - return _Shim(resp.text or "", getattr(resp, "response_id", "") or "") - if PROVIDER == "openrouter": - from src.commons_openrouter import invoke_openrouter - - return invoke_openrouter( - messages, - model=MODEL, - temperature=TEMPERATURE, - extra_body={"reasoning": {"effort": OPENROUTER_EFFORT}}, - ) - if PROVIDER == "fireworks": - from src.commons_fireworks import invoke_fireworks - - return invoke_fireworks( - messages, - model=MODEL, - reasoning_effort="none", - temperature=TEMPERATURE, - ) - return invoke_interfaze( - messages, - model=MODEL, - reasoning_effort=REASONING_EFFORT, - temperature=TEMPERATURE, - ) - - -async def process_example( - example: dict, rate_limiter: RateLimiter, writer: JsonlWriter, progress: dict -) -> dict | None: - instance_id = example["instance_id"] - prompt = build_prompt(example) - messages = [{"role": "user", "content": prompt}] - - last_error: str | None = None - for attempt in range(1, MAX_RETRIES + 1): - await rate_limiter.acquire() - start = time.perf_counter() - try: - response = await asyncio.to_thread(invoke_model, messages) - latency_ms = int((time.perf_counter() - start) * 1000) - content = (response.choices[0].message.content or "").strip() - request_id = getattr(response, "id", None) - if not content: - last_error = "empty response content" - raise RuntimeError(last_error) - - pred_sql = extract_sql(content) - # Persist per-instance .sql file so the official evaluate.py can - # consume the same directory if someone wants to cross-check. - PRED_SQL_DIR.mkdir(parents=True, exist_ok=True) - (PRED_SQL_DIR / f"{instance_id}.sql").write_text(pred_sql, encoding="utf-8") - - record = { - "instance_id": instance_id, - "db": example["db"], - "question": example["question"], - "external_knowledge": example.get("external_knowledge"), - "pred_sql": pred_sql, - "response": content, - "request_id": request_id, - "latency_ms": latency_ms, - "attempts": attempt, - } - await writer.append(record) - progress["done"] += 1 - tqdm.write( - f"[{progress['done']}/{progress['total']}] OK " - f"id={instance_id} db={example['db']} " - f"latency={latency_ms}ms sql_len={len(pred_sql)} " - f"req_id={request_id} attempt={attempt}" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - tqdm.write( - f"[error] id={instance_id} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(2 ** (attempt - 1)) - - progress["failed"] += 1 - tqdm.write(f"[FAILED] id={instance_id} after {MAX_RETRIES} attempts: {last_error}") - return None - - -async def run_prediction(examples: list[dict], limit: int | None): - done_ids = load_completed_ids(RESPONSES_PATH) - pending = [e for e in examples if e["instance_id"] not in done_ids] - if limit is not None: - pending = pending[:limit] - print(f"--limit applied: will run at most {limit} example(s)") - print( - f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {RESPONSES_PATH})" - ) - if not pending: - return - - writer = JsonlWriter(RESPONSES_PATH) - rate_limiter = RateLimiter(RATE_LIMIT) - progress = {"total": len(pending), "done": 0, "failed": 0} - tasks = [process_example(e, rate_limiter, writer, progress) for e in pending] - try: - await tqdm_asyncio.gather(*tasks, desc="spider2-lite/local") - except Exception: - traceback.print_exc() - print( - f"\nPrediction finished: {progress['done']}/{progress['total']} answered, " - f"{progress['failed']} failed." - ) - - -# ----------------------------------------------------------------------------- -# Evaluation — SQLite execution + row-set comparison. -# -# Ported from spider2-lite/evaluation_suite/evaluate.py. Kept faithful to the -# original semantics (column-vector matching, float tolerance of 1e-2, -# condition_cols / ignore_order flags, multi-gold). -# ----------------------------------------------------------------------------- - - -def _normalize(v): - return 0 if pd.isna(v) else v - - -def _sort_key(x): - return (x is None, str(x), isinstance(x, (int, float))) - - -def _vectors_match(v1, v2, ignore_order: bool, tol: float = 1e-2) -> bool: - v1 = [_normalize(x) for x in v1] - v2 = [_normalize(x) for x in v2] - if ignore_order: - v1 = sorted(v1, key=_sort_key) - v2 = sorted(v2, key=_sort_key) - if len(v1) != len(v2): - return False - for a, b in zip(v1, v2): - if pd.isna(a) and pd.isna(b): - continue - if isinstance(a, (int, float)) and isinstance(b, (int, float)): - if not math.isclose(float(a), float(b), abs_tol=tol): - return False - elif a != b: - return False - return True - - -def compare_table( - pred: pd.DataFrame, gold: pd.DataFrame, condition_cols, ignore_order: bool -) -> int: - if condition_cols: - if not isinstance(condition_cols, (list, tuple)): - condition_cols = [condition_cols] - gold_cols = gold.iloc[:, condition_cols] - else: - gold_cols = gold - t_gold = gold_cols.transpose().values.tolist() - t_pred = pred.transpose().values.tolist() - for gv in t_gold: - if not any(_vectors_match(gv, pv, ignore_order) for pv in t_pred): - return 0 - return 1 - - -def compare_multi( - pred: pd.DataFrame, - golds: list[pd.DataFrame], - multi_condition_cols, - ignore_order: bool, -) -> int: - if not golds: - return 0 - if multi_condition_cols in (None, [], [[]], [None]): - multi_condition_cols = [[] for _ in golds] - elif len(golds) > 1 and not all(isinstance(s, list) for s in multi_condition_cols): - multi_condition_cols = [multi_condition_cols for _ in golds] - for gold, cc in zip(golds, multi_condition_cols): - if compare_table(pred, gold, cc, ignore_order): - return 1 - return 0 - - -def resolve_gold_paths(instance_id: str) -> tuple[list[Path], bool]: - base = GOLD_EXEC_DIR / f"{instance_id}.csv" - if base.exists(): - return [base], True - pattern = re.compile(rf"^{re.escape(instance_id)}(_[a-z])?\.csv$") - matches = sorted( - GOLD_EXEC_DIR / name - for name in os.listdir(GOLD_EXEC_DIR) - if pattern.match(name) - ) - return matches, False - - -def execute_sqlite(db_path: Path, sql: str) -> tuple[bool, pd.DataFrame | str]: - """Run `sql` against `db_path`. Returns (ok, df-or-error-string). - - We copy the on-disk DB into :memory: (same pattern as the official - evaluate.py) — faster for repeated queries and isolates writes. - - Queries are capped at QUERY_TIMEOUT_S. A model can emit SQL that never - finishes (an unbounded cross join is the usual culprit), and without a cap - that single example hangs the whole scoring pass at 100% CPU. A query that - can't complete in the budget is scored as a failed execution, which is what - the official evaluate.py's func_timeout does too. - """ - try: - disk = sqlite3.connect(str(db_path)) - mem = sqlite3.connect(":memory:") - try: - disk.backup(mem) - deadline = time.monotonic() + QUERY_TIMEOUT_S - # Called every N VM instructions; non-zero aborts the query, which - # surfaces as sqlite3.OperationalError('interrupted'). - mem.set_progress_handler( - lambda: 1 if time.monotonic() > deadline else 0, 10_000 - ) - df = pd.read_sql_query(sql, mem) - mem.set_progress_handler(None, 0) - return True, df - finally: - mem.close() - disk.close() - except Exception as e: - return False, f"{type(e).__name__}: {e}" - - -def load_eval_standard() -> dict[str, dict]: - if not EVAL_STANDARD.exists(): - raise FileNotFoundError( - f"Missing {EVAL_STANDARD}. Run: uv run -m benchmarks.spider2_lite.fetch_data" - ) - out: dict[str, dict] = {} - with open(EVAL_STANDARD, encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - rec = json.loads(line) - out[rec["instance_id"]] = rec - return out - - -def evaluate_record(record: dict, eval_std: dict) -> dict: - instance_id = record["instance_id"] - db_path = SQLITE_DB_DIR / f"{record['db']}.sqlite" - if not db_path.exists(): - return { - "instance_id": instance_id, - "score": 0, - "error": f"missing sqlite db: {db_path}", - } - pred_sql = record.get("pred_sql") or "" - if not pred_sql.strip(): - return {"instance_id": instance_id, "score": 0, "error": "empty pred_sql"} - - ok, result = execute_sqlite(db_path, pred_sql) - if not ok: - return {"instance_id": instance_id, "score": 0, "error": f"sql error: {result}"} - - pred_df: pd.DataFrame = result # type: ignore[assignment] - gold_paths, is_single = resolve_gold_paths(instance_id) - if not gold_paths: - return {"instance_id": instance_id, "score": 0, "error": "no gold file"} - - standard = eval_std.get(instance_id, {}) - condition_cols = standard.get("condition_cols") - ignore_order = standard.get("ignore_order", False) - - try: - if is_single: - gold_df = pd.read_csv(gold_paths[0]) - score = compare_table(pred_df, gold_df, condition_cols, ignore_order) - else: - gold_dfs = [pd.read_csv(p) for p in gold_paths] - score = compare_multi(pred_df, gold_dfs, condition_cols, ignore_order) - except Exception as e: - return {"instance_id": instance_id, "score": 0, "error": f"compare: {e}"} - - return {"instance_id": instance_id, "score": score, "error": None} - - -def run_evaluation(total_local: int) -> None: - records = load_records(RESPONSES_PATH) - if not records: - print(f"No predictions at {RESPONSES_PATH}") - sys.exit(1) - eval_std = load_eval_standard() - - # Re-derive SQL from the stored response so --evaluate-only picks up - # extractor fixes without paying for new predictions (same convention as - # the mmmlu/gpqa runners re-running their answer parsers). - n_rewritten = 0 - for rec in records: - if not rec.get("response"): - continue - fresh = extract_sql(rec["response"]) - if fresh and fresh != rec.get("pred_sql"): - rec["pred_sql"] = fresh - n_rewritten += 1 - if n_rewritten: - print(f"Re-extracted SQL for {n_rewritten} record(s) with the current parser.") - - results: list[dict] = [] - for rec in tqdm(records, desc="Evaluating"): - res = evaluate_record(rec, eval_std) - results.append(res) - mark = "OK" if res["score"] == 1 else "X " - extra = f" ({res['error']})" if res.get("error") else "" - tqdm.write(f" {mark} {res['instance_id']}{extra}") - - correct = sum(r["score"] for r in results) - total = len(results) - accuracy_of_evaluated = correct / total if total else 0.0 - accuracy_of_subset = correct / total_local if total_local else 0.0 - - print(f"\n{'=' * 60}") - print( - f"Spider 2.0-Lite — SQLite subset ({PROVIDER}/{MODEL}, reasoning={REASONING_EFFORT})" - ) - print(f"{'=' * 60}") - print(f"Correct : {correct}/{total}") - print(f"Accuracy (of predicted): {accuracy_of_evaluated:.4f}") - print(f"Accuracy (of local 135): {accuracy_of_subset:.4f}") - - # Top error categories for quick eyeballing. - errors = [r for r in results if r["score"] == 0 and r.get("error")] - if errors: - print("\nTop error kinds:") - kinds: dict[str, int] = {} - for r in errors: - kind = (r["error"] or "").split(":", 1)[0] - kinds[kind] = kinds.get(kind, 0) + 1 - for k, v in sorted(kinds.items(), key=lambda kv: -kv[1]): - print(f" {v:4d} {k}") - - out = { - "accuracy_evaluated": accuracy_of_evaluated, - "accuracy_of_local_135": accuracy_of_subset, - "correct": correct, - "total_evaluated": total, - "total_local_subset": total_local, - "subset": "local (SQLite)", - "benchmark": "Spider 2.0-Lite", - "provider": PROVIDER, - "model": MODEL, - "reasoning_effort": "none" if PROVIDER == "fireworks" else REASONING_EFFORT, - "temperature": TEMPERATURE, - "per_example": results, - } - METRICS_PATH.parent.mkdir(parents=True, exist_ok=True) - with open(METRICS_PATH, "w") as f: - json.dump(out, f, indent=2) - print(f"\nMetrics saved to {METRICS_PATH}") - - -# ----------------------------------------------------------------------------- -# Entrypoint -# ----------------------------------------------------------------------------- - - -def set_paths() -> None: - """Per provider+model output paths so runs don't share a checkpoint. The - interfaze default keeps the original `spider2_lite_local` tag.""" - global TAG, RESPONSES_PATH, PRED_SQL_DIR, METRICS_PATH - if PROVIDER != "interfaze": - slug = re.sub(r"[^a-z0-9]+", "-", MODEL.rsplit("/", 1)[-1].lower()).strip("-") - TAG = f"spider2_lite_local_{PROVIDER}_{slug}" - if GEMINI_LEVEL != "auto": - TAG += f"_thinking{GEMINI_LEVEL}" - RESPONSES_PATH = RESULTS_DIR / f"{TAG}_responses.jsonl" - PRED_SQL_DIR = RESULTS_DIR / f"{TAG}_sql" - METRICS_PATH = RESULTS_DIR / f"{TAG}_metrics.json" - - -def main() -> None: - global PROVIDER, MODEL, GEMINI_LEVEL - parser = argparse.ArgumentParser( - description="Spider 2.0-Lite SQLite subset (Interfaze or Fireworks)" - ) - parser.add_argument( - "--provider", default="interfaze", choices=["interfaze", "fireworks", "openrouter", "gemini"] - ) - parser.add_argument( - "--thinking-level", - default="auto", - choices=["auto", "minimal", "low", "medium", "high"], - help="Gemini thinking_level; auto = the model's floor. Non-auto values " - "get their own output tag so a floor run is never overwritten.", - ) - parser.add_argument( - "--model", - default=None, - help="Model id. Defaults to interfaze-beta, or Inkling for --provider fireworks.", - ) - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument( - "--limit", - type=int, - default=None, - help="Only predict the first N pending examples", - ) - args = parser.parse_args() - - PROVIDER = args.provider - GEMINI_LEVEL = args.thinking_level - if args.model: - MODEL = args.model - elif PROVIDER == "fireworks": - MODEL = "accounts/fireworks/models/inkling" - elif PROVIDER == "openrouter": - MODEL = "thinkingmachines/inkling-small" - elif PROVIDER == "gemini": - MODEL = "gemini-3.7-flash" - set_paths() - - examples = load_local_examples() - print(f"Loaded {len(examples)} local (SQLite) examples from {ALL_EXAMPLES.name}") - - if args.evaluate_only: - run_evaluation(total_local=len(examples)) - elif args.predict_only: - asyncio.run(run_prediction(examples, limit=args.limit)) - else: - asyncio.run(run_prediction(examples, limit=args.limit)) - run_evaluation(total_local=len(examples)) - - -if __name__ == "__main__": - main() diff --git a/scripts/run_fireworks_suite.sh b/scripts/run_fireworks_suite.sh index fedce81..9720250 100755 --- a/scripts/run_fireworks_suite.sh +++ b/scripts/run_fireworks_suite.sh @@ -73,8 +73,11 @@ else echo " [skip] asr: set TARGET= (python -m bench_core list-targets)" fi -run_step spider2 uv run -m benchmarks.spider2_lite.spider2_lite \ - --provider "$PROVIDER" --model "$MODEL" +if [ -n "${TARGET:-}" ]; then + run_step spider2 uv run python -m bench_core run --benchmark spider2 --target "$TARGET" +else + echo " [skip] spider2: set TARGET= (python -m bench_core list-targets)" +fi if [ -n "${TARGET:-}" ]; then R="$([ "$REASONING" = "high" ] && echo high || echo off)" diff --git a/tests/test_spider2_bench.py b/tests/test_spider2_bench.py new file mode 100644 index 0000000..8f9bf5e --- /dev/null +++ b/tests/test_spider2_bench.py @@ -0,0 +1,72 @@ +"""Spider2-Lite: offline unit tests for the pure logic (SQL extraction, result +comparison). Full scoring re-executes SQL against the ~4GB gitignored data/, so a +score-replay is gated behind data/ presence + BENCH_SPIDER2_PARITY=1. +""" + +import json +import os +import pathlib + +import pytest + +from benchmarks.spider2_lite import bench + +RESULTS = pathlib.Path(__file__).resolve().parent.parent / "results" + + +def test_extract_sql_fenced(): + assert bench.extract_sql("```sql\nSELECT 1\n```") == "SELECT 1" + assert bench.extract_sql("prose\n```\nSELECT 2\n```\nmore") == "SELECT 2" + + +def test_extract_sql_unterminated_fence(): + # a model that opens ```sql, emits SQL, never closes the block + assert bench.extract_sql("Here you go:\n```sql\nSELECT 3;") == "SELECT 3;" + + +def test_extract_sql_no_fence_returns_whole(): + assert bench.extract_sql("SELECT 4") == "SELECT 4" + + +def test_vectors_match_float_tolerance_and_order(): + assert bench._vectors_match([1.0, 2.0], [1.005, 2.0], ignore_order=False) + assert not bench._vectors_match([1.0], [1.5], ignore_order=False) + + +def test_compare_table_column_match_unordered(): + import pandas as pd + + gold = pd.DataFrame({"a": [1, 2, 3]}) + pred = pd.DataFrame({"x": [3, 2, 1]}) + assert bench.compare_table(pred, gold, None, ignore_order=True) == 1 + assert bench.compare_table(pred, gold, None, ignore_order=False) == 0 + + +_PARITY_ON = bench._ALL_EXAMPLES.exists() and os.getenv("BENCH_SPIDER2_PARITY") + + +@pytest.mark.skipif( + not _PARITY_ON, reason="needs data/ + BENCH_SPIDER2_PARITY=1 (re-executes SQL)" +) +def test_full_scoring_parity_when_data_present(): + tag = "spider2_lite_local_fireworks_inkling" + resp_path = RESULTS / f"{tag}_responses.jsonl" + metrics_path = RESULTS / f"{tag}_metrics.json" + archived = json.loads(metrics_path.read_text()) + samples, records = [], [] + for line in resp_path.read_text().splitlines(): + if not line.strip(): + continue + rec = json.loads(line) + samples.append({"instance_id": rec["instance_id"], "db": rec["db"]}) + records.append( + { + "instance_id": rec["instance_id"], + "prediction": bench.extract_sql(rec["response"]), + } + ) + m = bench.score(records, samples) + # NOT exact: Spider2 re-executes SQL with a 120s timeout cap, so a heavy query + # near the boundary can flip between runs (warm cache vs cold). A faithful port + # reproduces the archived score within a small execution-noise tolerance. + assert abs(m["correct"] - archived["correct"]) <= 2 From 8ee79a581122c0d354f0fc8e371c6d9eefc84190 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 11:30:37 +0530 Subject: [PATCH 15/38] feat(bench_core): backfill legacy results + point report at the new contract (Steps 2+5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 2 — bench migrate-results: folds the 32 legacy flat results/*_metrics.json (+ responses/predictions) into results///. The contract keys by (benchmark, target), so colliding legacy runs (2-sample smokes, reasoning off/high, native-vs-openrouter) are deduped by most-evaluated-samples and the rest logged as shadowed (no silent drops). RefCOCO oracle folds in as a submetric; empty/None responses are left to resume, not counted done. Step 5 — report_scores.py now reads the new contract via bench_core.results .discover() (retiring the filename-prefix parsing, canon() shadowing, and smoke guards — the contract already resolved those). Renders all 8 tables unchanged; olmOCR still parsed from logs. 26 canonical metrics.json committed; the large regenerable per-run responses.jsonl are gitignored. 161 tests pass (+1 gated). --- .gitignore | 5 + bench_core/cli.py | 17 +++ bench_core/migrate.py | 171 +++++++++++++++++++++ scripts/report_scores.py | 310 ++++++++++++++++++++------------------- tests/test_migrate.py | 96 ++++++++++++ 5 files changed, 447 insertions(+), 152 deletions(-) create mode 100644 bench_core/migrate.py create mode 100644 tests/test_migrate.py diff --git a/.gitignore b/.gitignore index c338fdc..f4f3da8 100644 --- a/.gitignore +++ b/.gitignore @@ -216,6 +216,11 @@ external/ benchmarks/spider2_lite/data/ benchmarks/olmocr/bench/full_data/ +# New-contract per-run responses (large; regenerable via `bench migrate-results` +# from the committed legacy flat results, or re-produced by a fresh run). The +# canonical metrics.json under results/// IS committed. +results/*/*/responses.jsonl + # OCRBench spotting-eval scratch (generated during scoring) benchmarks/ocrbench_v2/eval_scripts/spotting_eval/gt/ benchmarks/ocrbench_v2/eval_scripts/spotting_eval/submit/ diff --git a/bench_core/cli.py b/bench_core/cli.py index 6486ecc..4fba0bb 100644 --- a/bench_core/cli.py +++ b/bench_core/cli.py @@ -132,6 +132,15 @@ def cmd_list_targets(args) -> None: print(f"{name:22} {t.provider:11} {t.model_id}") +def cmd_migrate_results(args) -> None: + from bench_core.migrate import migrate_results + + log = migrate_results(dry_run=args.dry_run) + for line in log: + print(line) + print(f"\n{'[dry-run] ' if args.dry_run else ''}{len(log)} action(s)") + + def main(argv=None) -> None: p = argparse.ArgumentParser(prog="bench") sub = p.add_subparsers(dest="cmd", required=True) @@ -155,6 +164,14 @@ def main(argv=None) -> None: lt = sub.add_parser("list-targets", help="list configured targets") lt.set_defaults(func=cmd_list_targets) + mr = sub.add_parser( + "migrate-results", help="backfill legacy flat results into the new contract" + ) + mr.add_argument( + "--dry-run", action="store_true", help="show actions without writing" + ) + mr.set_defaults(func=cmd_migrate_results) + args = p.parse_args(argv) args.func(args) diff --git a/bench_core/migrate.py b/bench_core/migrate.py new file mode 100644 index 0000000..f57b6dd --- /dev/null +++ b/bench_core/migrate.py @@ -0,0 +1,171 @@ +"""One-time backfill: fold the legacy flat results/*_metrics.json (+ their +responses/predictions siblings) into the new results/// +contract, so the report reads one layout and a partial run resumes rather than +re-bill. + +The new contract keys by (benchmark, target) only — it does not encode host or +reasoning — so when several legacy runs map to the same slot (a 2-sample smoke +vs a full run, or reasoning off vs high) the one with the most *evaluated* +samples wins and the rest are logged as shadowed. Idempotent; no silent drops. +""" + +from __future__ import annotations + +import json +from collections import defaultdict +from pathlib import Path + +from bench_core.results import DEFAULT_ROOT, RunStore, model_slug + +# count of samples actually evaluated (NOT a fixed denominator like Spider2's +# total_local_subset, which is 135 even for a 1-sample smoke) +_N_KEYS = ("num_samples", "total", "total_evaluated", "n") + + +def _classify(name: str) -> str | None: + if name.startswith("ocrbench_v2_"): + return "ocrbench_v2" + if name.startswith("mmmupro_standard_"): + return "mmmu_pro_standard" + if name.startswith("mmmupro_vision_"): + return "mmmu_pro_vision" + if name.startswith("mmmlulite_"): + return "mmmlu" + if name.startswith("refcoco_"): + return f"refcoco_{name.split('_')[1]}" + if name.startswith("voxpopuli_aa_"): + return "voxpopuli_aa" + if name.startswith("spider2_lite_local_"): + return "spider2_lite" + if "gpqa_diamond" in name: + return "gpqa" + return None + + +def _n(d: dict) -> int: + for k in _N_KEYS: + v = d.get(k) + if isinstance(v, int): + return v + return 0 + + +def _resp_shape(benchmark: str): + if benchmark == "spider2_lite": + return "instance_id", "pred_sql" + if benchmark.startswith("refcoco"): + return "id", "pred_bbox_xyxy" + return "id", "prediction" + + +def _migrate_responses( + metrics_path: Path, benchmark: str, store: RunStore +) -> str | None: + if benchmark == "ocrbench_v2": + src = metrics_path.with_name( + metrics_path.name.replace("_metrics.json", "_predictions.json") + ) + if not src.exists(): + return None + records = [ + { + "id": r["id"], + "response": r.get("predict", ""), + "prediction": r.get("predict", ""), + } + for r in json.loads(src.read_text()) + ] + else: + src = metrics_path.with_name( + metrics_path.name.replace("_metrics.json", "_responses.jsonl") + ) + if not src.exists(): + return None + id_key, pred_field = _resp_shape(benchmark) + records = [] + for line in src.read_text().splitlines(): + if line.strip(): + r = json.loads(line) + records.append( + { + id_key: r[id_key], + "response": r.get("response"), + "prediction": r.get(pred_field), + } + ) + # Only genuine answers migrate; empty/None responses are left to resume (a + # blank prediction must re-run, not count as done). + records = [r for r in records if (r.get("response") or "") != ""] + store._ensure_dir() + store.responses_path.write_text( + "".join(json.dumps(r, ensure_ascii=False) + "\n" for r in records) + ) + return f" responses: {src.name} ({len(records)} rows)" + + +def migrate_results( + results_dir: Path | str = DEFAULT_ROOT, dry_run: bool = False +) -> list[str]: + results_dir = Path(results_dir) + log: list[str] = [] + + groups: dict[tuple, list] = defaultdict(list) + for p in sorted(results_dir.glob("*_metrics.json")): + name = p.name + if "_oracle_" in name: # folded into the strict refcoco metrics + continue + benchmark = _classify(name) + if benchmark is None: + log.append(f"SKIP (unrecognized): {name}") + continue + d = json.loads(p.read_text()) + target = model_slug(d.get("model") or name) + groups[(benchmark, target)].append((p, d, _n(d))) + + for (benchmark, target), items in sorted(groups.items()): + winner = items[0] + for c in items[1:]: + if c[2] >= winner[2]: # tie -> later (sorted) wins, matching the old report + winner = c + p, d, n = winner + + newm = dict(d) + newm.update( + { + "benchmark": benchmark, + "target": target, + "provider": d.get("provider"), + "n": n, + "reasoning": {"mode": "high" if "high" in p.name else "off"}, + } + ) + if benchmark.startswith("refcoco"): + oracle_p = p.with_name( + p.name.replace("_metrics.json", "_oracle_metrics.json") + ) + if oracle_p.exists(): + od = json.loads(oracle_p.read_text()) + newm["oracle"] = { + "accuracy": od.get("accuracy"), + "mean_iou": od.get("mean_iou"), + "total": od.get("total"), + "interpretation_counts": od.get("interpretation_counts"), + } + + store = RunStore(benchmark, target, root=results_dir) + log.append( + f"metrics: {p.name} (n={n}) -> {benchmark}/{model_slug(target)}/metrics.json" + ) + if not dry_run: + store.write_metrics(newm) + r = _migrate_responses(p, benchmark, store) + if r: + log.append(r) + + for c in items: + if c is not winner: + log.append( + f" shadowed: {c[0].name} (n={c[2]}) — slot taken by the n={n} run" + ) + + return log diff --git a/scripts/report_scores.py b/scripts/report_scores.py index 5fe27a7..8d28784 100644 --- a/scripts/report_scores.py +++ b/scripts/report_scores.py @@ -7,27 +7,23 @@ uv run python scripts/report_scores.py --list # just the model names uv run python scripts/report_scores.py --model inkling --tsv # paste into Sheets -Reads results/*_metrics.json plus the olmOCR run logs (olmOCR-bench prints its -per-split table to stdout instead of persisting metrics, so the logs are the -only source for those numbers). - -Model names are canonicalised across providers — accounts/fireworks/models/inkling, -thinkingmachines/inkling and google/gemini-3.7-flash all collapse to the bare -model name, so one row means one model regardless of which host produced it. -Where the same model+benchmark exists from two hosts, both are kept and the -provider is shown, because the hosts are NOT equivalent at the same nominal -reasoning setting. +Reads the results///metrics.json contract (written by the CLI +and `bench migrate-results`) plus the olmOCR run logs (olmOCR-bench prints its +per-split table to stdout instead of persisting metrics, so the logs are the only +source for those numbers). One row per target; the contract keys by +(benchmark, target), so `bench migrate-results` already picked the best run per +slot when several legacy runs collided. """ from __future__ import annotations import argparse -import json import re import sys from pathlib import Path ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) # so `bench_core` is importable when run as a script RESULTS = ROOT / "results" LOGS = ROOT / "logs" @@ -84,17 +80,73 @@ "HI_IN", ] -MMMU_STD_SUBJECTS = """overall Art Electronics Economics Marketing Finance Art_Theory -Public_Health Basic_Medical_Science Sociology Literature Physics Energy_and_Power History -Design Materials Biology Psychology Geography Manage Pharmacy Agriculture Clinical_Medicine -Accounting Computer_Science Architecture_and_Engineering Chemistry Math -Diagnostics_and_Laboratory_Medicine Mechanical_Engineering Music""".split() +MMMU_STD_SUBJECTS = [ + "overall", + "Art", + "Electronics", + "Economics", + "Marketing", + "Finance", + "Art_Theory", + "Public_Health", + "Basic_Medical_Science", + "Sociology", + "Literature", + "Physics", + "Energy_and_Power", + "History", + "Design", + "Materials", + "Biology", + "Psychology", + "Geography", + "Manage", + "Pharmacy", + "Agriculture", + "Clinical_Medicine", + "Accounting", + "Computer_Science", + "Architecture_and_Engineering", + "Chemistry", + "Math", + "Diagnostics_and_Laboratory_Medicine", + "Mechanical_Engineering", + "Music", +] -MMMU_VIS_SUBJECTS = """overall Economics Art_Theory Basic_Medical_Science Art Literature -Pharmacy Clinical_Medicine Sociology Public_Health Design Physics History Chemistry -Electronics Marketing Geography Math Biology Computer_Science Manage Finance Agriculture -Accounting Psychology Mechanical_Engineering Diagnostics_and_Laboratory_Medicine -Energy_and_Power Materials Architecture_and_Engineering Music""".split() +MMMU_VIS_SUBJECTS = [ + "overall", + "Economics", + "Art_Theory", + "Basic_Medical_Science", + "Art", + "Literature", + "Pharmacy", + "Clinical_Medicine", + "Sociology", + "Public_Health", + "Design", + "Physics", + "History", + "Chemistry", + "Electronics", + "Marketing", + "Geography", + "Math", + "Biology", + "Computer_Science", + "Manage", + "Finance", + "Agriculture", + "Accounting", + "Psychology", + "Mechanical_Engineering", + "Diagnostics_and_Laboratory_Medicine", + "Energy_and_Power", + "Materials", + "Architecture_and_Engineering", + "Music", +] # --------------------------------------------------------------------------- @@ -119,141 +171,91 @@ def _pct(x): def load_metrics() -> dict: - """{model: {benchmark_key: payload}} from results/*_metrics.json.""" - out: dict[str, dict] = {} + """{model: {benchmark_key: payload}} from the results/// + metrics.json contract (written by the CLI and `bench migrate-results`).""" + from bench_core.results import discover - def _n(payload): - """Sample count, for choosing between duplicate runs of the same - benchmark. Different hosts write different filenames for the same model - (voxpopuli_aa_gemini_* vs voxpopuli_aa_openrouter_google_*), and a 2-row - smoke test must never shadow a full run just because its filename sorts - later.""" - for k in ("n", "num_samples", "total", "total_local_subset"): - v = payload.get(k) - if isinstance(v, int): - return v - return 0 - - def put(model, key, payload, source=""): - bucket = out.setdefault(canon(model), {}) - prev = bucket.get(key) - payload = dict(payload, source=source) - if prev is None or _n(payload) >= _n(prev): - if prev is not None and _n(prev): - payload["shadowed"] = (prev.get("source"), _n(prev)) - bucket[key] = payload - else: - prev.setdefault("shadowed", (source, _n(payload))) - - for p in sorted(RESULTS.glob("*_metrics.json")): - name = p.name - try: - d = json.loads(p.read_text()) - except json.JSONDecodeError: + out: dict[str, dict] = {} + for d in discover(RESULTS): + benchmark, model = d.get("benchmark"), d.get("target") + if not benchmark or not model: continue - model = d.get("model") or "" - - if name.startswith("ocrbench_v2_"): - model = model or name.replace("ocrbench_v2_", "").replace( - "_metrics.json", "" - ) - covered = sum(1 for v in d.get("en_scores", {}).values() if v.get("count")) - put( - model, - "ocrbench", - { - "en_overall": _pct(d.get("en_overall")), - "cn_overall": _pct(d.get("cn_overall")), - "cats": { - k: _pct(v["avg"]) if v.get("count") else None - for k, v in d.get("en_scores", {}).items() - }, - "counts": { - k: v.get("count", 0) for k, v in d.get("en_scores", {}).items() - }, - "partial": covered < 8, - "covered": covered, - }, - ) - elif name.startswith("mmmupro_"): - setting = "standard" if "_standard_" in name else "vision" - reasoning = "high" if "reasoninghigh" in name else "off" - put( - model, - f"mmmupro_{setting}_{reasoning}", - { - "overall": _pct(d.get("accuracy")), - "n": d.get("num_samples"), - "subjects": { - k: _pct(v["accuracy"]) - for k, v in (d.get("per_subject") or {}).items() - }, - "reasoning": reasoning, + b = out.setdefault(model, {}) + reasoning = (d.get("reasoning") or {}).get("mode", "off") + + if benchmark == "gpqa": + b["gpqa"] = { + "overall": _pct(d.get("accuracy")), + "n": d.get("total") or d.get("n"), + "domains": { + k: _pct(v["accuracy"]) + for k, v in (d.get("per_domain") or {}).items() }, - ) - elif name.startswith("mmmlulite_"): - put( - model, - "mmmlu", - { - "macro": _pct(d.get("macro_accuracy")), - "n": d.get("num_samples"), - "langs": { - k: _pct(v["accuracy"]) - for k, v in (d.get("per_language") or {}).items() - }, - "reasoning": "high" if "reasoninghigh" in name else "off", - }, - ) - elif name.startswith("refcoco_"): - split = name.split("_")[1] - oracle = "_oracle_" in name - put( - model, - f"refcoco_{split}" + ("_oracle" if oracle else ""), - { - "acc": _pct(d.get("accuracy")), - "mean_iou": round(d.get("mean_iou", 0), 4), - "n": d.get("total"), - "split": split, - }, - ) - elif name.startswith("voxpopuli_aa_"): + } + elif benchmark == "voxpopuli_aa": wer = d.get("corpus_wer") - put( - model, - "asr", - { - "wer": _pct(wer), - "inv": _pct(1 - wer) if wer is not None else None, - "cer": _pct(d.get("corpus_cer")), - "n": d.get("num_samples"), + b["asr"] = { + "wer": _pct(wer), + "inv": _pct(1 - wer) if wer is not None else None, + "cer": _pct(d.get("corpus_cer")), + "n": d.get("num_samples") or d.get("n"), + } + elif benchmark == "mmmlu": + b["mmmlu"] = { + "macro": _pct(d.get("macro_accuracy")), + "n": d.get("num_samples") or d.get("n"), + "langs": { + k: _pct(v["accuracy"]) + for k, v in (d.get("per_language") or {}).items() }, - ) - elif name.startswith("spider2_lite_local_"): - put( - model, - "spider2", - { - "acc": _pct(d.get("accuracy_of_local_135")), - "correct": d.get("correct"), - "n": d.get("total_local_subset"), + "reasoning": reasoning, + } + elif benchmark.startswith("mmmu_pro_"): + setting = benchmark[len("mmmu_pro_") :] + b[f"mmmupro_{setting}_{reasoning}"] = { + "overall": _pct(d.get("accuracy")), + "n": d.get("num_samples") or d.get("n"), + "subjects": { + k: _pct(v["accuracy"]) + for k, v in (d.get("per_subject") or {}).items() }, - ) - elif "gpqa_diamond" in name: - model = model or name.split("_")[1] - put( - model, - "gpqa", - { - "overall": _pct(d.get("accuracy")), - "n": d.get("total"), - "domains": { - k: _pct(v["accuracy"]) - for k, v in (d.get("per_domain") or {}).items() - }, + "reasoning": reasoning, + } + elif benchmark.startswith("refcoco_"): + split = benchmark[len("refcoco_") :] + b[f"refcoco_{split}"] = { + "acc": _pct(d.get("accuracy")), + "mean_iou": round(d.get("mean_iou", 0), 4), + "n": d.get("total") or d.get("n"), + "split": split, + } + orc = d.get("oracle") + if orc: + b[f"refcoco_{split}_oracle"] = { + "acc": _pct(orc.get("accuracy")), + "mean_iou": round(orc.get("mean_iou", 0), 4), + "n": orc.get("total") or d.get("total"), + "split": split, + } + elif benchmark == "ocrbench_v2": + en = d.get("en_scores", {}) + covered = sum(1 for v in en.values() if v.get("count")) + b["ocrbench"] = { + "en_overall": _pct(d.get("en_overall")), + "cn_overall": _pct(d.get("cn_overall")), + "cats": { + k: _pct(v["avg"]) if v.get("count") else None for k, v in en.items() }, - ) + "counts": {k: v.get("count", 0) for k, v in en.items()}, + "partial": covered < 8, + "covered": covered, + } + elif benchmark == "spider2_lite": + b["spider2"] = { + "acc": _pct(d.get("accuracy_of_local_135")), + "correct": d.get("correct"), + "n": d.get("total_local_subset"), + } return out @@ -268,16 +270,20 @@ def load_olmocr() -> dict: for log in dict.fromkeys(candidates): text = log.read_text(errors="ignore").replace("\r", "\n") head = re.search( - r"^(\S+)\s*:\s*Average Score:\s*([\d.]+)%\s*±\s*([\d.]+)%", text, re.M + r"^(\S+)\s*:\s*Average Score:\s*([\d.]+)%\s*±\s*([\d.]+)%", + text, + re.MULTILINE, ) if not head: continue model, overall, ci = head.group(1), float(head.group(2)), float(head.group(3)) splits = { m.group(1): float(m.group(2)) - for m in re.finditer(r"^\s+(\w+)\.jsonl\s*:\s*([\d.]+)%", text, re.M) + for m in re.finditer( + r"^\s+(\w+)\.jsonl\s*:\s*([\d.]+)%", text, re.MULTILINE + ) } - base = re.search(r"^\s+baseline\s*:\s*([\d.]+)%", text, re.M) + base = re.search(r"^\s+baseline\s*:\s*([\d.]+)%", text, re.MULTILINE) if base: splits["baseline"] = float(base.group(1)) out[canon(model)] = { diff --git a/tests/test_migrate.py b/tests/test_migrate.py new file mode 100644 index 0000000..65dd949 --- /dev/null +++ b/tests/test_migrate.py @@ -0,0 +1,96 @@ +"""Backfill: legacy flat results -> new contract, with (benchmark, target) +dedup (full run beats a smoke) and refcoco oracle merged as a submetric. +""" + +import json + +from bench_core.migrate import migrate_results + + +def _write(root, name, obj): + (root / name).write_text(json.dumps(obj)) + + +def test_dedup_full_run_beats_smoke(tmp_path): + _write( + tmp_path, + "fireworks_inkling_thinkingoff_gpqa_diamond_metrics.json", + {"model": "accounts/fireworks/models/inkling", "accuracy": 0.87, "total": 198}, + ) + ( + tmp_path / "fireworks_inkling_thinkingoff_gpqa_diamond_responses.jsonl" + ).write_text(json.dumps({"id": "a", "response": "A", "prediction": "A"}) + "\n") + # a 2-sample smoke of the SAME model -> same (gpqa, inkling) slot + _write( + tmp_path, + "smoke_inkling_gpqa_diamond_metrics.json", + {"model": "accounts/fireworks/models/inkling", "accuracy": 0.5, "total": 2}, + ) + + log = migrate_results(tmp_path) + metrics = json.loads((tmp_path / "gpqa" / "inkling" / "metrics.json").read_text()) + assert metrics["benchmark"] == "gpqa" + assert metrics["target"] == "inkling" + assert metrics["accuracy"] == 0.87 # full run won, not the smoke + assert metrics["n"] == 198 + assert any("shadowed" in line and "n=2" in line for line in log) + + +def test_refcoco_oracle_merged_as_submetric(tmp_path): + _write( + tmp_path, + "refcoco_val_fireworks_x_metrics.json", + { + "model": "accounts/fireworks/models/inkling", + "accuracy": 0.32, + "mean_iou": 0.3, + "total": 10, + }, + ) + _write( + tmp_path, + "refcoco_val_fireworks_x_oracle_metrics.json", + {"accuracy": 0.81, "mean_iou": 0.7, "total": 10}, + ) + (tmp_path / "refcoco_val_fireworks_x_responses.jsonl").write_text( + json.dumps({"id": "1", "response": "[1,2,3,4]", "pred_bbox_xyxy": [1, 2, 3, 4]}) + + "\n" + ) + + migrate_results(tmp_path) + m = json.loads((tmp_path / "refcoco_val" / "inkling" / "metrics.json").read_text()) + assert m["accuracy"] == 0.32 + assert m["oracle"]["accuracy"] == 0.81 + # response migrated with the new prediction field for resume/scoring + rows = [ + json.loads(x) + for x in (tmp_path / "refcoco_val" / "inkling" / "responses.jsonl") + .read_text() + .splitlines() + ] + assert rows[0]["prediction"] == [1, 2, 3, 4] + + +def test_empty_responses_not_migrated(tmp_path): + _write( + tmp_path, + "voxpopuli_aa_x_metrics.json", + { + "model": "accounts/fireworks/models/inkling", + "corpus_wer": 0.1, + "num_samples": 2, + }, + ) + (tmp_path / "voxpopuli_aa_x_responses.jsonl").write_text( + json.dumps({"id": "1", "response": "hello", "prediction": "hello"}) + + "\n" + + json.dumps({"id": "2", "response": "", "prediction": ""}) + + "\n" + ) + migrate_results(tmp_path) + rows = ( + (tmp_path / "voxpopuli_aa" / "inkling" / "responses.jsonl") + .read_text() + .splitlines() + ) + assert len(rows) == 1 # the empty-response row is left to resume From b32683faf56414b53861bed1bc6a5ef140eed4a1 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 11:54:27 +0530 Subject: [PATCH 16/38] =?UTF-8?q?ci(bench=5Fcore):=20GitHub=20Actions=20?= =?UTF-8?q?=E2=80=94=20offline=20test=20gate=20+=20benchmark=20runner=20(S?= =?UTF-8?q?tage=206)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit .github/workflows/tests.yml: pytest + ruff on every push/PR (test_targets.py validates new targets). .github/workflows/benchmark.yml: workflow_dispatch (target / benchmarks / sample_size / reasoning) + nightly schedule over ci_regression targets; matrix from scripts/ci_matrix.py; per-run metrics artifact; report job diffs vs baseline (scripts/ci_compare.py) into the job summary. Defaults to cheap --sample smokes (full runs opt-in). CI set is the API-only benchmarks; olmocr (poppler/chromium/full-dataset) and spider2 (4GB data/) are left to local/self-hosted. Adding a model = a targets.yaml entry + its secret. inkling + gemini-3.7-flash flagged ci_regression. README documents it all. 165 tests pass (+1 gated). --- .github/workflows/benchmark.yml | 128 ++++++++++++++++++++++++++++++++ .github/workflows/tests.yml | 27 +++++++ README.md | 36 +++++++-- bench_core/targets.yaml | 2 + scripts/ci_compare.py | 116 +++++++++++++++++++++++++++++ scripts/ci_matrix.py | 53 +++++++++++++ tests/test_ci_matrix.py | 36 +++++++++ 7 files changed, 392 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/benchmark.yml create mode 100644 .github/workflows/tests.yml create mode 100644 scripts/ci_compare.py create mode 100644 scripts/ci_matrix.py create mode 100644 tests/test_ci_matrix.py diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..2ad900c --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,128 @@ +name: benchmark + +# Runs benchmarks through the unified CLI. Manual dispatch picks a target + +# benchmarks; the nightly schedule smoke-tests every `ci_regression: true` target +# to catch a provider changing its API (an adapter breaking shows up as a crash +# or empty-content, not a score drift). Defaults to small --sample runs; a full +# run is opt-in via the sample_size input. +# +# Adding a model to automation: add an entry to bench_core/targets.yaml (mark it +# ci_regression: true for the nightly), and set its provider's secret. No code. +# +# Not in CI: olmocr (needs poppler + playwright + the full dataset to score) and +# spider2 (needs the ~4GB data/). Run those locally / on a self-hosted runner. + +on: + workflow_dispatch: + inputs: + target: + description: "target name (see bench_core/targets.yaml)" + default: inkling + benchmarks: + description: "comma-separated benchmarks, or 'all' for the CI set" + default: all + sample_size: + description: "samples per benchmark (blank = full run — slow/expensive)" + default: "5" + reasoning: + description: "reasoning override: off|low|medium|high (blank = benchmark default)" + default: "" + schedule: + - cron: "0 6 * * *" # nightly plumbing health check + +permissions: + contents: read + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.m.outputs.matrix }} + targets: ${{ steps.m.outputs.targets }} + sample: ${{ steps.m.outputs.sample }} + reasoning: ${{ steps.m.outputs.reasoning }} + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + - run: uv sync + - id: m + env: + EVENT_NAME: ${{ github.event_name }} + INPUT_TARGET: ${{ github.event.inputs.target }} + INPUT_BENCHMARKS: ${{ github.event.inputs.benchmarks }} + run: | + MATRIX="$(uv run python scripts/ci_matrix.py)" + echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" + echo "targets=$(printf '%s' "$MATRIX" | uv run python -c 'import json,sys; print(" ".join(sorted({i["target"] for i in json.load(sys.stdin)["include"]})))')" >> "$GITHUB_OUTPUT" + echo "sample=${{ github.event.inputs.sample_size || '5' }}" >> "$GITHUB_OUTPUT" + echo "reasoning=${{ github.event.inputs.reasoning }}" >> "$GITHUB_OUTPUT" + + run: + needs: plan + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + INTERFAZE_API_KEY: ${{ secrets.INTERFAZE_API_KEY }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + - run: uv sync + - name: Run ${{ matrix.benchmark }} on ${{ matrix.target }} + run: | + SAMPLE="${{ needs.plan.outputs.sample }}" + REASONING="${{ needs.plan.outputs.reasoning }}" + uv run python -m bench_core run \ + --target "${{ matrix.target }}" --benchmark "${{ matrix.benchmark }}" \ + ${SAMPLE:+--sample "$SAMPLE"} ${REASONING:+--reasoning "$REASONING"} + - name: Upload metrics + uses: actions/upload-artifact@v4 + with: + name: metrics-${{ matrix.target }}-${{ matrix.benchmark }} + path: results/${{ matrix.benchmark }}*/${{ matrix.target }}/metrics.json + if-no-files-found: ignore + + report: + needs: [plan, run] + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + - run: uv sync + - name: Fetch run metrics + uses: actions/download-artifact@v4 + with: + path: incoming + merge-multiple: true + - name: Overlay fresh metrics onto the contract + run: | + # artifacts store their original results///metrics.json path + if [ -d incoming/results ]; then cp -r incoming/results/* results/; fi + - name: Compare vs baseline + write summary + run: | + for t in ${{ needs.plan.outputs.targets }}; do + uv run python scripts/ci_compare.py --target "$t" >> "$GITHUB_STEP_SUMMARY" + done + { + echo "" + echo '
Full score tables' + echo "" + echo '```' + uv run python scripts/report_scores.py --all 2>&1 || true + echo '```' + echo '
' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..5c82fd5 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,27 @@ +name: tests + +# The free, fast gate on every push/PR: the offline bench_core suite + target +# validation (test_targets.py loads/validates every bench_core/targets.yaml entry, +# so adding a model is checked here). No secrets, no API calls. + +on: + push: + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + - name: Install deps + run: uv sync --group dev + - name: Lint (new code) + run: uv run ruff check bench_core tests + - name: Tests (offline) + run: uv run pytest -q diff --git a/README.md b/README.md index 4ced58d..2bfdec0 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,11 @@ FIREWORKS_API_KEY=... JIGSAWSTACK_API_KEY=... # only for benchmarks/obj_detection/ob_det_api.py ``` -Every runner accepts `--limit N` for a smoke test and `--evaluate-only` / -`--predict-only` to split prediction and scoring. Re-running a benchmark -resumes from its checkpoint file — already-completed samples are skipped. +Benchmarks run through one CLI: `uv run python -m bench_core run --target +--benchmark ` (see `python -m bench_core list-targets`). `--sample N` is a +smoke test; re-running resumes from the per-run checkpoint (completed samples are +skipped). The remaining bespoke runners (`ob_det_api`, `olmocr_bench_reducto`, +BFCL) keep their own `--limit`/`--evaluate-only` flags. Two extra setup steps for specific benchmarks: @@ -143,9 +145,6 @@ Links: [paper](https://arxiv.org/abs/2501.00321) · [repo](https://github.com/Yu uv run python -m bench_core run --target gpt-5.5 --benchmark ocrbench_v2 uv run python -m bench_core run --target gemini-3.7-flash --benchmark ocrbench_v2 uv run python -m bench_core run --target inkling --benchmark ocrbench_v2 - -# Evaluate without re-running predictions -uv run -m benchmarks.ocrbench_v2.ocrbench_v2 --evaluate-only ``` --- @@ -264,3 +263,28 @@ uv run -m benchmarks.spider2_lite.fetch_data # downloads the ~4GB data/ (requi uv run python -m bench_core run --target gpt-5.5 --benchmark spider2 uv run python -m bench_core run --target inkling --benchmark spider2 ``` + +## Automation (GitHub Actions) + +Two workflows under `.github/workflows/`: + +- **`tests.yml`** — the free gate on every push/PR: the offline `bench_core` suite + + lint. `test_targets.py` validates every `bench_core/targets.yaml` entry, so a + new model is checked here before it ever runs. +- **`benchmark.yml`** — runs benchmarks through `python -m bench_core run`. + - *Manual* (`workflow_dispatch`): pick `target`, `benchmarks` (csv or `all`), + `sample_size` (blank = full run), and an optional `reasoning` override. + - *Nightly* (`schedule`): smoke-runs every target flagged `ci_regression: true` + to catch a provider changing its API (an adapter breaking shows up as a crash, + not a silent drift). + - A `report` job diffs each run against the committed baseline + (`scripts/ci_compare.py`) and writes the score tables to the job summary. + +**Secrets required:** `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GEMINI_API_KEY`, +`FIREWORKS_API_KEY`, `OPENROUTER_API_KEY`, `INTERFAZE_API_KEY`, `HF_TOKEN`. + +**Add a model to automation:** add an entry to `bench_core/targets.yaml` (mark it +`ci_regression: true` for the nightly) and set the provider's secret — no code. + +**Not in CI:** `olmocr` (needs poppler + playwright + the full dataset to score) +and `spider2` (needs the ~4GB `data/`); run those locally or on a self-hosted runner. diff --git a/bench_core/targets.yaml b/bench_core/targets.yaml index 3028c7a..b2dc8aa 100644 --- a/bench_core/targets.yaml +++ b/bench_core/targets.yaml @@ -3,6 +3,7 @@ targets: inkling: provider: fireworks model_id: accounts/fireworks/models/inkling + ci_regression: true deepseek-v4-flash: provider: fireworks @@ -37,6 +38,7 @@ targets: gemini-3.7-flash: provider: gemini model_id: gemini-3.7-flash + ci_regression: true gemini-2.5-flash: provider: gemini diff --git a/scripts/ci_compare.py b/scripts/ci_compare.py new file mode 100644 index 0000000..fb7a7f2 --- /dev/null +++ b/scripts/ci_compare.py @@ -0,0 +1,116 @@ +"""Compare a fresh run's metrics against the committed baseline and emit a +markdown summary (for $GITHUB_STEP_SUMMARY). Regressions beyond a tolerance are +flagged as ::warning:: — non-blocking, since benchmark scores aren't +deterministic (reasoning models, SQL-timeout boundaries). + + uv run python scripts/ci_compare.py --target inkling [--tolerance 0.02] + +Baseline = the metrics.json at git HEAD for the same path; "fresh" = the working +tree after the run. Smoke runs (n far below baseline) are reported but not judged. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +RESULTS = ROOT / "results" + +# primary metric per benchmark: (key, higher_is_better) +_PRIMARY = [ + ("corpus_wer", False), + ("accuracy_of_local_135", True), + ("macro_accuracy", True), + ("en_overall", True), + ("accuracy", True), +] + + +def primary(metrics: dict): + for key, higher in _PRIMARY: + if key in metrics and isinstance(metrics[key], (int, float)): + return key, metrics[key], higher + return None, None, True + + +def _baseline(path: Path) -> dict | None: + rel = path.relative_to(ROOT) + try: + out = subprocess.run( + ["git", "show", f"HEAD:{rel.as_posix()}"], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + return json.loads(out.stdout) + except (subprocess.CalledProcessError, json.JSONDecodeError): + return None + + +def compare_target(target: str, tolerance: float) -> tuple[list[list], list[str]]: + rows, warnings = [], [] + for mpath in sorted(RESULTS.glob(f"*/{target}/metrics.json")): + fresh = json.loads(mpath.read_text()) + base = _baseline(mpath) + key, new, higher = primary(fresh) + bench = fresh.get("benchmark", mpath.parent.parent.name) + n = fresh.get("n") + if base is None: + rows.append([bench, key or "-", _fmt(new), "new", "—"]) + continue + _, old, _ = primary(base) + delta = (new - old) if (new is not None and old is not None) else None + base_n = base.get("n") + note = "" + if base_n and n and n < base_n * 0.5: + note = f"smoke (n={n} vs {base_n})" + elif delta is not None: + regressed = (delta < -tolerance) if higher else (delta > tolerance) + if regressed: + note = "REGRESSION" + warnings.append( + f"{bench}/{target}: {key} {old:.4f} -> {new:.4f} (Δ{delta:+.4f})" + ) + rows.append( + [ + bench, + key or "-", + _fmt(new), + _fmt(old), + _fmt(delta, sign=True) + (f" {note}" if note else ""), + ] + ) + return rows, warnings + + +def _fmt(v, sign=False): + if not isinstance(v, (int, float)): + return "—" + return f"{v:+.4f}" if sign else f"{v:.4f}" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--target", required=True) + ap.add_argument("--tolerance", type=float, default=0.02) + args = ap.parse_args() + + rows, warnings = compare_target(args.target, args.tolerance) + print(f"### Benchmark results — `{args.target}` vs baseline\n") + if not rows: + print("_no metrics found for this target_") + return + print("| benchmark | metric | new | baseline | Δ |") + print("|---|---|---|---|---|") + for r in rows: + print("| " + " | ".join(str(c) for c in r) + " |") + for w in warnings: + print(f"\n::warning::regression — {w}") + + +if __name__ == "__main__": + main() diff --git a/scripts/ci_matrix.py b/scripts/ci_matrix.py new file mode 100644 index 0000000..c75efa2 --- /dev/null +++ b/scripts/ci_matrix.py @@ -0,0 +1,53 @@ +"""Emit the GitHub Actions job matrix ({"include": [{target, benchmark}, ...]}). + +- schedule -> every target flagged `ci_regression: true` x the CI benchmark set +- dispatch -> the chosen target x the chosen benchmarks (or the CI set for "all") + +The CI set is the API-only, --sample-friendly benchmarks. olmocr (needs poppler + +playwright + the full dataset to score) and spider2 (needs the ~4GB data/) are +left to local/self-hosted runs. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +CI_BENCHMARKS = ["gpqa", "mmmlu", "mmmu_pro", "asr", "ocrbench_v2", "refcoco"] + + +def build_matrix( + event: str, target: str | None = None, benchmarks: str | None = None +) -> dict: + if event == "schedule": + from bench_core.config import load_all_targets + + targets = [ + n for n, t in load_all_targets().items() if t.raw.get("ci_regression") + ] + targets = targets or ["inkling"] + benches = CI_BENCHMARKS + else: # workflow_dispatch (or manual) + targets = [target or "inkling"] + b = (benchmarks or "").strip() + benches = ( + CI_BENCHMARKS + if (not b or b == "all") + else [x.strip() for x in b.split(",") if x.strip()] + ) + return { + "include": [{"target": t, "benchmark": b} for t in targets for b in benches] + } + + +if __name__ == "__main__": + matrix = build_matrix( + os.getenv("EVENT_NAME", "workflow_dispatch"), + os.getenv("INPUT_TARGET") or None, + os.getenv("INPUT_BENCHMARKS") or None, + ) + print(json.dumps(matrix)) diff --git a/tests/test_ci_matrix.py b/tests/test_ci_matrix.py new file mode 100644 index 0000000..7394809 --- /dev/null +++ b/tests/test_ci_matrix.py @@ -0,0 +1,36 @@ +"""CI matrix builder: dispatch (target x chosen benchmarks) and schedule (every +ci_regression target x the CI benchmark set).""" + +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "scripts")) + +from ci_matrix import CI_BENCHMARKS, build_matrix + + +def test_dispatch_specific_benchmarks(): + m = build_matrix("workflow_dispatch", "inkling", "gpqa,asr") + assert m["include"] == [ + {"target": "inkling", "benchmark": "gpqa"}, + {"target": "inkling", "benchmark": "asr"}, + ] + + +def test_dispatch_all_expands_to_ci_set(): + m = build_matrix("workflow_dispatch", "gpt-5.5", "all") + assert [i["benchmark"] for i in m["include"]] == CI_BENCHMARKS + assert {i["target"] for i in m["include"]} == {"gpt-5.5"} + + +def test_dispatch_default_target(): + m = build_matrix("workflow_dispatch", None, None) + assert {i["target"] for i in m["include"]} == {"inkling"} + + +def test_schedule_uses_ci_regression_targets(): + m = build_matrix("schedule") + targets = {i["target"] for i in m["include"]} + # inkling + gemini-3.7-flash are flagged ci_regression in targets.yaml + assert "inkling" in targets and "gemini-3.7-flash" in targets + assert all(i["benchmark"] in CI_BENCHMARKS for i in m["include"]) From e5af3843b1e2e4ac109785fbb54859e11179620b Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 16:22:25 +0530 Subject: [PATCH 17/38] ci: interactive add-target workflow + run benchmarks on merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add-target.yml: a GitHub-UI form (name/provider/model_id/capabilities_json/ ci_regression) -> scripts/add_target.py validates the entry (loads, resolves caps, builds adapter) and appends it to bench_core/targets.yaml -> opens a PR. Inputs passed via env (no shell injection through the JSON). benchmark.yml: new push trigger on main touching targets.yaml runs a FULL benchmark for the target(s) whose entry changed (ci_matrix.changed_targets diffs the merge). So both paths — the guided form and a hand-edited YAML PR — run the benchmark only once the PR is merged. Action versions bumped to match tests.yml. Adds tests for add_target (append/validate/reject) and changed_targets. 170 tests pass (+1 gated). --- .github/workflows/add-target.yml | 67 +++++++++++++++++++ .github/workflows/benchmark.yml | 26 ++++++-- .github/workflows/tests.yml | 4 +- README.md | 19 ++++-- scripts/add_target.py | 108 +++++++++++++++++++++++++++++++ scripts/ci_matrix.py | 78 +++++++++++++++------- tests/test_add_target.py | 74 +++++++++++++++++++++ 7 files changed, 340 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/add-target.yml create mode 100644 scripts/add_target.py create mode 100644 tests/test_add_target.py diff --git a/.github/workflows/add-target.yml b/.github/workflows/add-target.yml new file mode 100644 index 0000000..ccfe6d5 --- /dev/null +++ b/.github/workflows/add-target.yml @@ -0,0 +1,67 @@ +name: add-target + +# Interactive "add a model" from the GitHub UI (Actions -> add-target -> Run +# workflow). Fill in the fields; the workflow validates the entry, appends it to +# bench_core/targets.yaml, and opens a PR. It does NOT run the benchmark — that +# happens when the PR is merged (see benchmark.yml's push trigger). This is the +# same outcome as hand-editing targets.yaml in a PR; just a guided form instead. + +on: + workflow_dispatch: + inputs: + name: + description: "target name (map key, e.g. my-model)" + required: true + provider: + description: "provider" + required: true + type: choice + options: [fireworks, openrouter, openai, anthropic, gemini, interfaze] + model_id: + description: "provider model id (e.g. accounts/fireworks/models/my-model)" + required: true + capabilities_json: + description: 'optional capability overrides as JSON, e.g. {"reasoning":{"style":"thinking_budget","off_value":0,"on_value":-1,"true_off":true}}' + required: false + default: "" + ci_regression: + description: "include in the nightly regression run" + type: boolean + default: false + +permissions: + contents: write + pull-requests: write + +jobs: + add: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v9.0.0 + with: + enable-cache: true + - run: uv sync + - name: Validate + append target + env: # via env, not inline ${{ }}, so JSON quotes can't break/inject the shell + T_NAME: ${{ github.event.inputs.name }} + T_PROVIDER: ${{ github.event.inputs.provider }} + T_MODEL: ${{ github.event.inputs.model_id }} + T_CAPS: ${{ github.event.inputs.capabilities_json }} + T_CI: ${{ github.event.inputs.ci_regression }} + run: | + ARGS=(--name "$T_NAME" --provider "$T_PROVIDER" --model-id "$T_MODEL" --capabilities-json "$T_CAPS") + [ "$T_CI" = "true" ] && ARGS+=(--ci-regression) + uv run python scripts/add_target.py "${ARGS[@]}" + - name: Open PR + uses: peter-evans/create-pull-request@v7 + with: + branch: add-target/${{ github.event.inputs.name }} + title: "Add benchmark target: ${{ github.event.inputs.name }}" + commit-message: "feat(targets): add ${{ github.event.inputs.name }}" + body: | + Adds `${{ github.event.inputs.name }}` (${{ github.event.inputs.provider }} / `${{ github.event.inputs.model_id }}`) to `bench_core/targets.yaml`. + + The entry was validated (loads, resolves capabilities, builds its adapter). + **Merging this PR triggers the benchmark run for the new target** (benchmark.yml push trigger). + add-paths: bench_core/targets.yaml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 2ad900c..2f3f06b 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -29,6 +29,9 @@ on: default: "" schedule: - cron: "0 6 * * *" # nightly plumbing health check + push: + branches: [main] + paths: [bench_core/targets.yaml] # a merged target change runs a full benchmark for it permissions: contents: read @@ -42,8 +45,10 @@ jobs: sample: ${{ steps.m.outputs.sample }} reasoning: ${{ steps.m.outputs.reasoning }} steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 + - uses: actions/checkout@v6 + with: + fetch-depth: 0 # need the merge's parent to diff targets.yaml on push + - uses: astral-sh/setup-uv@v9.0.0 with: enable-cache: true - run: uv sync @@ -53,10 +58,17 @@ jobs: INPUT_TARGET: ${{ github.event.inputs.target }} INPUT_BENCHMARKS: ${{ github.event.inputs.benchmarks }} run: | + if [ "${{ github.event_name }}" = "push" ]; then + git show "${{ github.event.before }}:bench_core/targets.yaml" > /tmp/old_targets.yaml 2>/dev/null || : > /tmp/old_targets.yaml + export OLD_TARGETS_FILE=/tmp/old_targets.yaml + SAMPLE="" # a merged target change gets a full benchmark run + else + SAMPLE="${{ github.event.inputs.sample_size || '5' }}" + fi MATRIX="$(uv run python scripts/ci_matrix.py)" echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" echo "targets=$(printf '%s' "$MATRIX" | uv run python -c 'import json,sys; print(" ".join(sorted({i["target"] for i in json.load(sys.stdin)["include"]})))')" >> "$GITHUB_OUTPUT" - echo "sample=${{ github.event.inputs.sample_size || '5' }}" >> "$GITHUB_OUTPUT" + echo "sample=$SAMPLE" >> "$GITHUB_OUTPUT" echo "reasoning=${{ github.event.inputs.reasoning }}" >> "$GITHUB_OUTPUT" run: @@ -74,8 +86,8 @@ jobs: INTERFAZE_API_KEY: ${{ secrets.INTERFAZE_API_KEY }} HF_TOKEN: ${{ secrets.HF_TOKEN }} steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v9.0.0 with: enable-cache: true - run: uv sync @@ -98,8 +110,8 @@ jobs: if: always() runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v9.0.0 with: enable-cache: true - run: uv sync diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5c82fd5..bac68ca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,8 +15,8 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v9.0.0 with: enable-cache: true - name: Install deps diff --git a/README.md b/README.md index 2bfdec0..0dfe728 100644 --- a/README.md +++ b/README.md @@ -266,7 +266,7 @@ uv run python -m bench_core run --target inkling --benchmark spider2 ## Automation (GitHub Actions) -Two workflows under `.github/workflows/`: +Three workflows under `.github/workflows/`: - **`tests.yml`** — the free gate on every push/PR: the offline `bench_core` suite + lint. `test_targets.py` validates every `bench_core/targets.yaml` entry, so a @@ -277,14 +277,25 @@ Two workflows under `.github/workflows/`: - *Nightly* (`schedule`): smoke-runs every target flagged `ci_regression: true` to catch a provider changing its API (an adapter breaking shows up as a crash, not a silent drift). + - *On merge* (`push` to `main` touching `targets.yaml`): diffs the merge and runs + a **full** benchmark for the target(s) that changed. - A `report` job diffs each run against the committed baseline (`scripts/ci_compare.py`) and writes the score tables to the job summary. +- **`add-target.yml`** — the interactive "add a model" form (Actions → add-target → + Run workflow): enter `name` / `provider` / `model_id` / optional + `capabilities_json` / `ci_regression`. It validates the entry, appends it to + `targets.yaml`, and **opens a PR** — it does not run anything itself. + +**Two ways to add a model, both gated on merge:** +1. *Guided:* run `add-target` from the UI → it opens a PR. +2. *By hand:* edit `bench_core/targets.yaml` in a PR. + +Either way, `tests.yml` validates the entry on the PR, and **merging the PR** is +what triggers `benchmark.yml` to run the new target (full run). No code, and the +benchmark only runs once a human has merged. **Secrets required:** `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GEMINI_API_KEY`, `FIREWORKS_API_KEY`, `OPENROUTER_API_KEY`, `INTERFAZE_API_KEY`, `HF_TOKEN`. -**Add a model to automation:** add an entry to `bench_core/targets.yaml` (mark it -`ci_regression: true` for the nightly) and set the provider's secret — no code. - **Not in CI:** `olmocr` (needs poppler + playwright + the full dataset to score) and `spider2` (needs the ~4GB `data/`); run those locally or on a self-hosted runner. diff --git a/scripts/add_target.py b/scripts/add_target.py new file mode 100644 index 0000000..206268f --- /dev/null +++ b/scripts/add_target.py @@ -0,0 +1,108 @@ +"""Append a target to bench_core/targets.yaml from UI/CLI inputs, then validate. + +Used by the add-target GitHub workflow (which opens a PR) and locally: + + uv run python scripts/add_target.py --name my-model --provider fireworks \\ + --model-id accounts/fireworks/models/my-model \\ + --capabilities-json '{"reasoning":{"style":"effort","off_value":"none","on_value":"high","true_off":false}}' \\ + --ci-regression + +The entry is appended as text (preserving the file's comments), then the whole +file is re-parsed and the new target must resolve to typed capabilities + build +its adapter — otherwise nothing is written and it exits non-zero. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import tempfile +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) + +from bench_core.config import ( + PROVIDERS, + build_adapter, + load_all_targets, + resolve_capabilities, +) + +TARGETS_FILE = ROOT / "bench_core" / "targets.yaml" + + +def _entry_text(name, provider, model_id, capabilities, ci_regression) -> str: + lines = [f" {name}:", f" provider: {provider}", f" model_id: {model_id}"] + if ci_regression: + lines.append(" ci_regression: true") + if capabilities: + block = yaml.safe_dump( + {"capabilities": capabilities}, sort_keys=False, default_flow_style=False + ) + lines += [" " + ln for ln in block.rstrip("\n").splitlines()] + return "\n".join(lines) + "\n" + + +def add_target( + name, + provider, + model_id, + capabilities_json="", + ci_regression=False, + path=TARGETS_FILE, +) -> None: + path = Path(path) + if provider not in PROVIDERS: + raise SystemExit(f"unknown provider {provider!r}; known: {sorted(PROVIDERS)}") + existing = load_all_targets(path) + if name in existing: + raise SystemExit(f"target {name!r} already exists — edit it directly instead") + capabilities = json.loads(capabilities_json) if capabilities_json.strip() else None + + current = path.read_text() + new_text = ( + current.rstrip("\n") + + "\n" + + _entry_text(name, provider, model_id, capabilities, ci_regression) + ) + + # validate on a temp copy before touching the real file + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as tf: + tf.write(new_text) + tmp = Path(tf.name) + try: + targets = load_all_targets(tmp) + if name not in targets: + raise SystemExit(f"target {name!r} did not parse from the appended entry") + resolve_capabilities(targets[name]) # merges + type-checks capabilities + build_adapter(targets[name]) # provider resolves to an adapter + finally: + tmp.unlink(missing_ok=True) + + path.write_text(new_text) + print(f"added target {name!r} ({provider} / {model_id}) to {path}") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--name", required=True) + ap.add_argument("--provider", required=True, choices=sorted(PROVIDERS)) + ap.add_argument("--model-id", required=True) + ap.add_argument("--capabilities-json", default="") + ap.add_argument("--ci-regression", action="store_true") + args = ap.parse_args() + add_target( + args.name, + args.provider, + args.model_id, + args.capabilities_json, + args.ci_regression, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci_matrix.py b/scripts/ci_matrix.py index c75efa2..4b66281 100644 --- a/scripts/ci_matrix.py +++ b/scripts/ci_matrix.py @@ -1,7 +1,8 @@ """Emit the GitHub Actions job matrix ({"include": [{target, benchmark}, ...]}). -- schedule -> every target flagged `ci_regression: true` x the CI benchmark set -- dispatch -> the chosen target x the chosen benchmarks (or the CI set for "all") +- schedule -> every target flagged `ci_regression: true` x the CI benchmark set +- dispatch -> the chosen target x the chosen benchmarks (or the CI set for "all") +- push -> the target(s) whose entry changed in this merge x the CI set The CI set is the API-only, --sample-friendly benchmarks. olmocr (needs poppler + playwright + the full dataset to score) and spider2 (needs the ~4GB data/) are @@ -20,34 +21,65 @@ CI_BENCHMARKS = ["gpqa", "mmmlu", "mmmu_pro", "asr", "ocrbench_v2", "refcoco"] +def _matrix(targets, benches): + return { + "include": [{"target": t, "benchmark": b} for t in targets for b in benches] + } + + +def changed_targets(old_path: str | None, new_path: str) -> list[str]: + """Target names added or modified between two targets.yaml files (merge diff).""" + import yaml + + def load(p): + if not p or not Path(p).exists(): + return {} + return (yaml.safe_load(Path(p).read_text()) or {}).get("targets") or {} + + old, new = load(old_path), load(new_path) + return [name for name, spec in new.items() if old.get(name) != spec] + + def build_matrix( - event: str, target: str | None = None, benchmarks: str | None = None + event: str, + target: str | None = None, + benchmarks: str | None = None, + targets: list[str] | None = None, ) -> dict: if event == "schedule": from bench_core.config import load_all_targets - targets = [ + flagged = [ n for n, t in load_all_targets().items() if t.raw.get("ci_regression") ] - targets = targets or ["inkling"] - benches = CI_BENCHMARKS - else: # workflow_dispatch (or manual) - targets = [target or "inkling"] - b = (benchmarks or "").strip() - benches = ( - CI_BENCHMARKS - if (not b or b == "all") - else [x.strip() for x in b.split(",") if x.strip()] - ) - return { - "include": [{"target": t, "benchmark": b} for t in targets for b in benches] - } + return _matrix(flagged or ["inkling"], CI_BENCHMARKS) + if event == "push": + return _matrix(targets or [], CI_BENCHMARKS) + # workflow_dispatch (or manual) + b = (benchmarks or "").strip() + benches = ( + CI_BENCHMARKS + if (not b or b == "all") + else [x.strip() for x in b.split(",") if x.strip()] + ) + return _matrix([target or "inkling"], benches) if __name__ == "__main__": - matrix = build_matrix( - os.getenv("EVENT_NAME", "workflow_dispatch"), - os.getenv("INPUT_TARGET") or None, - os.getenv("INPUT_BENCHMARKS") or None, - ) - print(json.dumps(matrix)) + event = os.getenv("EVENT_NAME", "workflow_dispatch") + if event == "push": + tl = changed_targets( + os.getenv("OLD_TARGETS_FILE"), + os.getenv("NEW_TARGETS_FILE", "bench_core/targets.yaml"), + ) + print(json.dumps(build_matrix("push", targets=tl))) + else: + print( + json.dumps( + build_matrix( + event, + os.getenv("INPUT_TARGET") or None, + os.getenv("INPUT_BENCHMARKS") or None, + ) + ) + ) diff --git a/tests/test_add_target.py b/tests/test_add_target.py new file mode 100644 index 0000000..0df5db2 --- /dev/null +++ b/tests/test_add_target.py @@ -0,0 +1,74 @@ +"""Interactive add-target: append a validated entry to targets.yaml, and the +merge-diff helper that decides which targets a push should benchmark. +""" + +import pathlib +import sys + +import pytest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "scripts")) + +from add_target import add_target +from ci_matrix import changed_targets + +from bench_core.capabilities import ReasoningStyle +from bench_core.config import load_all_targets, resolve_capabilities + + +def _base(tmp_path): + p = tmp_path / "targets.yaml" + p.write_text( + "targets:\n x:\n provider: fireworks\n model_id: accounts/fireworks/models/x\n" + ) + return p + + +def test_append_and_validate(tmp_path): + p = _base(tmp_path) + add_target("newmodel", "gemini", "gemini-3.7-flash", path=p) + targets = load_all_targets(p) + assert "x" in targets and "newmodel" in targets # existing preserved + assert targets["newmodel"].provider == "gemini" + resolve_capabilities(targets["newmodel"]) # resolves without error + + +def test_append_with_capabilities_json(tmp_path): + p = _base(tmp_path) + add_target( + "g25", + "gemini", + "gemini-2.5-flash", + capabilities_json='{"reasoning":{"style":"thinking_budget","off_value":0,"on_value":-1,"true_off":true}}', + ci_regression=True, + path=p, + ) + t = load_all_targets(p)["g25"] + assert t.raw.get("ci_regression") is True + caps = resolve_capabilities(t) + assert caps.reasoning.style is ReasoningStyle.THINKING_BUDGET + assert caps.reasoning.off_value == 0 + + +def test_rejects_unknown_provider(tmp_path): + with pytest.raises(SystemExit, match="unknown provider"): + add_target("z", "telepathy", "m", path=_base(tmp_path)) + + +def test_rejects_duplicate(tmp_path): + with pytest.raises(SystemExit, match="already exists"): + add_target("x", "fireworks", "y", path=_base(tmp_path)) + + +def test_changed_targets_detects_added_and_modified(tmp_path): + old = tmp_path / "old.yaml" + new = tmp_path / "new.yaml" + old.write_text("targets:\n a:\n provider: fireworks\n model_id: x\n") + new.write_text( + "targets:\n a:\n provider: fireworks\n model_id: x\n" + " b:\n provider: gemini\n model_id: y\n" + ) + assert changed_targets(str(old), str(new)) == ["b"] # only the added one + + new.write_text("targets:\n a:\n provider: openai\n model_id: x\n") + assert changed_targets(str(old), str(new)) == ["a"] # modified entry From 743372cda90aad28b4ed45ca05e6dc1b8a49c98f Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 16:35:38 +0530 Subject: [PATCH 18/38] =?UTF-8?q?ci:=20no=20default=20model=20=E2=80=94=20?= =?UTF-8?q?nothing=20runs=20unless=20explicitly=20selected/flagged?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit benchmark.yml: dispatch 'target' is now required (no inkling default), so an accidental Run-workflow can't spend on a model you didn't pick. ci_matrix drops both inkling fallbacks: dispatch with no target and schedule with no ci_regression target both yield an empty matrix. A 'has_work' plan output gates the run job so an empty matrix cleanly produces zero jobs. 170 tests pass. --- .github/workflows/add-target.yml | 6 ------ .github/workflows/benchmark.yml | 19 +++++-------------- scripts/ci_matrix.py | 8 +++++--- tests/test_ci_matrix.py | 6 +++--- 4 files changed, 13 insertions(+), 26 deletions(-) diff --git a/.github/workflows/add-target.yml b/.github/workflows/add-target.yml index ccfe6d5..fe26e74 100644 --- a/.github/workflows/add-target.yml +++ b/.github/workflows/add-target.yml @@ -1,11 +1,5 @@ name: add-target -# Interactive "add a model" from the GitHub UI (Actions -> add-target -> Run -# workflow). Fill in the fields; the workflow validates the entry, appends it to -# bench_core/targets.yaml, and opens a PR. It does NOT run the benchmark — that -# happens when the PR is merged (see benchmark.yml's push trigger). This is the -# same outcome as hand-editing targets.yaml in a PR; just a guided form instead. - on: workflow_dispatch: inputs: diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 2f3f06b..d1bc67f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,23 +1,11 @@ name: benchmark -# Runs benchmarks through the unified CLI. Manual dispatch picks a target + -# benchmarks; the nightly schedule smoke-tests every `ci_regression: true` target -# to catch a provider changing its API (an adapter breaking shows up as a crash -# or empty-content, not a score drift). Defaults to small --sample runs; a full -# run is opt-in via the sample_size input. -# -# Adding a model to automation: add an entry to bench_core/targets.yaml (mark it -# ci_regression: true for the nightly), and set its provider's secret. No code. -# -# Not in CI: olmocr (needs poppler + playwright + the full dataset to score) and -# spider2 (needs the ~4GB data/). Run those locally / on a self-hosted runner. - on: workflow_dispatch: inputs: target: - description: "target name (see bench_core/targets.yaml)" - default: inkling + description: "target name — required (see bench_core/targets.yaml / list-targets)" + required: true benchmarks: description: "comma-separated benchmarks, or 'all' for the CI set" default: all @@ -42,6 +30,7 @@ jobs: outputs: matrix: ${{ steps.m.outputs.matrix }} targets: ${{ steps.m.outputs.targets }} + has_work: ${{ steps.m.outputs.has_work }} sample: ${{ steps.m.outputs.sample }} reasoning: ${{ steps.m.outputs.reasoning }} steps: @@ -68,11 +57,13 @@ jobs: MATRIX="$(uv run python scripts/ci_matrix.py)" echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" echo "targets=$(printf '%s' "$MATRIX" | uv run python -c 'import json,sys; print(" ".join(sorted({i["target"] for i in json.load(sys.stdin)["include"]})))')" >> "$GITHUB_OUTPUT" + echo "has_work=$(printf '%s' "$MATRIX" | uv run python -c 'import json,sys; print(str(bool(json.load(sys.stdin)["include"])).lower())')" >> "$GITHUB_OUTPUT" echo "sample=$SAMPLE" >> "$GITHUB_OUTPUT" echo "reasoning=${{ github.event.inputs.reasoning }}" >> "$GITHUB_OUTPUT" run: needs: plan + if: needs.plan.outputs.has_work == 'true' # no target selected/flagged -> nothing runs runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/scripts/ci_matrix.py b/scripts/ci_matrix.py index 4b66281..13cced4 100644 --- a/scripts/ci_matrix.py +++ b/scripts/ci_matrix.py @@ -52,17 +52,19 @@ def build_matrix( flagged = [ n for n, t in load_all_targets().items() if t.raw.get("ci_regression") ] - return _matrix(flagged or ["inkling"], CI_BENCHMARKS) + return _matrix( + flagged, CI_BENCHMARKS + ) # explicit opt-in only; nothing flagged -> nothing runs if event == "push": return _matrix(targets or [], CI_BENCHMARKS) - # workflow_dispatch (or manual) + # workflow_dispatch (or manual): no target -> nothing runs (no default model) b = (benchmarks or "").strip() benches = ( CI_BENCHMARKS if (not b or b == "all") else [x.strip() for x in b.split(",") if x.strip()] ) - return _matrix([target or "inkling"], benches) + return _matrix([target] if target else [], benches) if __name__ == "__main__": diff --git a/tests/test_ci_matrix.py b/tests/test_ci_matrix.py index 7394809..b021f5f 100644 --- a/tests/test_ci_matrix.py +++ b/tests/test_ci_matrix.py @@ -23,9 +23,9 @@ def test_dispatch_all_expands_to_ci_set(): assert {i["target"] for i in m["include"]} == {"gpt-5.5"} -def test_dispatch_default_target(): - m = build_matrix("workflow_dispatch", None, None) - assert {i["target"] for i in m["include"]} == {"inkling"} +def test_dispatch_no_target_runs_nothing(): + # no model selected -> empty matrix -> zero jobs (no default model) + assert build_matrix("workflow_dispatch", None, None)["include"] == [] def test_schedule_uses_ci_regression_targets(): From b5f2a129212faf94e1b9a17dfb7312dd36945681 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 16:44:03 +0530 Subject: [PATCH 19/38] =?UTF-8?q?ci:=20audit=20=E2=80=94=20eliminate=20acc?= =?UTF-8?q?idental=20and=20unnecessary=20runs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tests.yml: was on every branch push AND double-ran PRs (push + pull_request). Now push only on main (feature branches covered by their PR) + a concurrency group with cancel-in-progress so stale commits don't pile up runs. - changed_targets: an empty diff baseline (first push / force-push / unavailable) used to treat EVERY target as changed and benchmark the whole registry — now returns nothing. And a ci_regression flag flip used to trigger a full paid run though the model is unchanged — now ignored (it only affects nightly membership). - benchmark.yml: report job no longer spins a runner on a no-op push (gated on has_work); a concurrency group queues (doesn't cancel) paid runs per ref so two passes never run at once; nightly dropped from daily to weekly (Mon), and still only runs ci_regression-flagged targets. 172 tests pass (+1 gated). --- .github/workflows/benchmark.yml | 11 +++++++++-- .github/workflows/tests.yml | 10 ++++++---- bench_core/capabilities.py | 2 +- bench_core/cli.py | 2 -- bench_core/config.py | 6 +----- bench_core/decode.py | 2 +- bench_core/errors.py | 6 ++---- scripts/ci_matrix.py | 18 ++++++++++++++++-- tests/test_add_target.py | 19 +++++++++++++++++++ 9 files changed, 55 insertions(+), 21 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d1bc67f..a0f4319 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -16,11 +16,17 @@ on: description: "reasoning override: off|low|medium|high (blank = benchmark default)" default: "" schedule: - - cron: "0 6 * * *" # nightly plumbing health check + - cron: "0 6 * * 1" # weekly (Mon) plumbing health check — only ci_regression targets push: branches: [main] paths: [bench_core/targets.yaml] # a merged target change runs a full benchmark for it +# never run two paid benchmark passes for the same ref at once (queue, don't cancel +# — a half-finished expensive run's spend shouldn't be thrown away) +concurrency: + group: benchmark-${{ github.ref }} + cancel-in-progress: false + permissions: contents: read @@ -98,7 +104,8 @@ jobs: report: needs: [plan, run] - if: always() + # only when something actually ran; skip the runner entirely on a no-op push + if: always() && needs.plan.outputs.has_work == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bac68ca..be7e71d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,13 +1,15 @@ name: tests -# The free, fast gate on every push/PR: the offline bench_core suite + target -# validation (test_targets.py loads/validates every bench_core/targets.yaml entry, -# so adding a model is checked here). No secrets, no API calls. - on: push: + branches: [main] # post-merge sanity; feature branches are covered by their PR pull_request: +# supersede stale runs on the same ref instead of piling them up +concurrency: + group: tests-${{ github.ref }} + cancel-in-progress: true + permissions: contents: read diff --git a/bench_core/capabilities.py b/bench_core/capabilities.py index fc62af7..17d20e5 100644 --- a/bench_core/capabilities.py +++ b/bench_core/capabilities.py @@ -1,4 +1,4 @@ -"""Declared model capabilities + the merge that resolves them.""" +# Declared model capabilities + the merge that resolves them. from __future__ import annotations diff --git a/bench_core/cli.py b/bench_core/cli.py index 4fba0bb..c82b806 100644 --- a/bench_core/cli.py +++ b/bench_core/cli.py @@ -1,5 +1,3 @@ -"""bench CLI: run a benchmark against a target, or list targets.""" - from __future__ import annotations import argparse diff --git a/bench_core/config.py b/bench_core/config.py index ec99c09..c45c420 100644 --- a/bench_core/config.py +++ b/bench_core/config.py @@ -1,5 +1,3 @@ -"""Targets + provider registry + the capability merge chain.""" - from __future__ import annotations import os @@ -16,7 +14,6 @@ # --- provider registry ---------------------------------------------------- - @dataclass class ProviderSpec: adapter_cls: type @@ -130,8 +127,7 @@ class ProviderSpec: } -# --- targets --------------------------------------------------------------- - +# targets @dataclass class Target: diff --git a/bench_core/decode.py b/bench_core/decode.py index cb1c4f9..2b44568 100644 --- a/bench_core/decode.py +++ b/bench_core/decode.py @@ -1,4 +1,4 @@ -"""Normalize a host response into `Response` (unifies the three shapes).""" +# Normalize a host response into `Response` (unifies the three shapes). from __future__ import annotations diff --git a/bench_core/errors.py b/bench_core/errors.py index 2301824..23a36d3 100644 --- a/bench_core/errors.py +++ b/bench_core/errors.py @@ -1,5 +1,3 @@ -"""Classify a provider exception into the signal the fallback ladder acts on.""" - from __future__ import annotations import re @@ -9,9 +7,9 @@ class ErrorKind(str, Enum): PARAM_REJECTED = ( - "param_rejected" # a param the model won't accept -> adjust + retry + "param_rejected" ) - RATE_LIMITED = "rate_limited" # 429 -> backoff + retry + RATE_LIMITED = "rate_limited" TRANSIENT = "transient" # timeout / 5xx / connection -> backoff + retry EMPTY_CONTENT = "empty_content" # returned nothing usable -> retry FATAL = "fatal" # auth / missing model / unrecognized 4xx -> fail the sample diff --git a/scripts/ci_matrix.py b/scripts/ci_matrix.py index 13cced4..d13bde2 100644 --- a/scripts/ci_matrix.py +++ b/scripts/ci_matrix.py @@ -28,7 +28,8 @@ def _matrix(targets, benches): def changed_targets(old_path: str | None, new_path: str) -> list[str]: - """Target names added or modified between two targets.yaml files (merge diff).""" + """Target names whose *run-affecting* spec was added or modified between two + targets.yaml files (the merge diff).""" import yaml def load(p): @@ -37,7 +38,20 @@ def load(p): return (yaml.safe_load(Path(p).read_text()) or {}).get("targets") or {} old, new = load(old_path), load(new_path) - return [name for name, spec in new.items() if old.get(name) != spec] + if not old: + # No usable baseline (first push / force-push / unavailable). Don't + # assume everything changed and benchmark the whole registry — run + # nothing; the author can dispatch explicitly. + return [] + + def runspec(spec): + # ci_regression only controls nightly membership, not the run itself, + # so flipping it must NOT trigger a (full, paid) benchmark. + return {k: v for k, v in (spec or {}).items() if k != "ci_regression"} + + return [ + n for n, spec in new.items() if n not in old or runspec(old[n]) != runspec(spec) + ] def build_matrix( diff --git a/tests/test_add_target.py b/tests/test_add_target.py index 0df5db2..ceb2ba3 100644 --- a/tests/test_add_target.py +++ b/tests/test_add_target.py @@ -72,3 +72,22 @@ def test_changed_targets_detects_added_and_modified(tmp_path): new.write_text("targets:\n a:\n provider: openai\n model_id: x\n") assert changed_targets(str(old), str(new)) == ["a"] # modified entry + + +def test_changed_targets_empty_baseline_runs_nothing(tmp_path): + # first push / force-push / no old file -> don't benchmark the whole registry + new = tmp_path / "new.yaml" + new.write_text("targets:\n a:\n provider: fireworks\n model_id: x\n") + assert changed_targets(None, str(new)) == [] + assert changed_targets(str(tmp_path / "missing.yaml"), str(new)) == [] + + +def test_changed_targets_ignores_ci_regression_flip(tmp_path): + # toggling ci_regression must NOT trigger a (paid) benchmark — model unchanged + old = tmp_path / "old.yaml" + new = tmp_path / "new.yaml" + old.write_text("targets:\n a:\n provider: fireworks\n model_id: x\n") + new.write_text( + "targets:\n a:\n provider: fireworks\n model_id: x\n ci_regression: true\n" + ) + assert changed_targets(str(old), str(new)) == [] From 7c5046c54a52437375ee2d3537a00ff54eef2792 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 16:53:36 +0530 Subject: [PATCH 20/38] fix(benchmarks): restore full MMMLU + RefCOCO+/g variants (don't lite-ify) The migration had narrowed two benchmarks to their default variant only, dropping functionality (scores of the defaults were unchanged, but the options were gone): - MMMLU: was lite-only. Restores --variant lite|full (full = openai/MMMLU ~196k); lite stays the default so the archived leaderboard numbers are unchanged. The result dir is now mmmlu_lite (like mmmu_pro_standard); backfill + report updated. - RefCOCO: was base-only. --variant now also accepts plus-* (RefCOCO+) and g-* (RefCOCOg); base splits (val/testA/testB/test) unchanged. Every other benchmark already runs the full dataset (--sample is only a smoke flag). Adds unit tests for both variant builders; report re-migrated. 174 tests pass (+1 gated). --- README.md | 15 ++++---- bench_core/errors.py | 2 +- bench_core/migrate.py | 4 ++- bench_core/reasoning.py | 9 ----- benchmarks/mmmlu/bench.py | 57 ++++++++++++++++++++++--------- benchmarks/obj_detection/bench.py | 31 +++++++++++++++-- scripts/report_scores.py | 26 ++++++++------ tests/test_mmmlu_bench.py | 36 +++++++++++++++++++ tests/test_refcoco_bench.py | 7 ++++ 9 files changed, 139 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 0dfe728..94983ac 100644 --- a/README.md +++ b/README.md @@ -172,11 +172,13 @@ Links: [RefCOCO/RefCOCO+ paper](https://arxiv.org/abs/1608.00272) · [RefCOCOg p Metric: Acc@IoU=0.5 on the referring-expression bounding box. ```bash -# Any target; --variant is the split (val | testA | testB | test). Metrics include -# a strict Acc@0.5 plus a format-tolerant `oracle` upper bound (submetrics.oracle). +# --variant selects dataset + split: base RefCOCO (val|testA|testB|test), or +# RefCOCO+ (plus-val|plus-testA|plus-testB) / RefCOCOg (g-val|g-test). Metrics +# include a strict Acc@0.5 plus a format-tolerant `oracle` upper bound. uv run python -m bench_core run --target gpt-5.5 --benchmark refcoco --variant val uv run python -m bench_core run --target gemini-3.7-flash --benchmark refcoco --variant testA -uv run python -m bench_core run --target inkling --benchmark refcoco --variant val +uv run python -m bench_core run --target inkling --benchmark refcoco --variant plus-testB +uv run python -m bench_core run --target inkling --benchmark refcoco --variant g-val # JigsawStack object_detection API (a bespoke endpoint, not a chat model) uv run -m benchmarks.obj_detection.ob_det_api @@ -206,10 +208,11 @@ Links: [dataset](https://huggingface.co/datasets/openai/MMMLU) 14 languages, exact-match accuracy macro-averaged across languages. ```bash -# Any target (MMMLU-lite, 14 languages, macro-averaged) -uv run python -m bench_core run --target gpt-5.5 --benchmark mmmlu +# 14 languages, macro-averaged. --variant lite (opencompass/mmmlu_lite, ~20k, +# DEFAULT, matches the archived leaderboard numbers) or full (openai/MMMLU, ~196k). +uv run python -m bench_core run --target gpt-5.5 --benchmark mmmlu # lite uv run python -m bench_core run --target gemini-3.7-flash --benchmark mmmlu --reasoning high -uv run python -m bench_core run --target inkling --benchmark mmmlu +uv run python -m bench_core run --target inkling --benchmark mmmlu --variant full # full MMLU ``` --- diff --git a/bench_core/errors.py b/bench_core/errors.py index 23a36d3..f3c41ed 100644 --- a/bench_core/errors.py +++ b/bench_core/errors.py @@ -11,7 +11,7 @@ class ErrorKind(str, Enum): ) RATE_LIMITED = "rate_limited" TRANSIENT = "transient" # timeout / 5xx / connection -> backoff + retry - EMPTY_CONTENT = "empty_content" # returned nothing usable -> retry + EMPTY_CONTENT = "empty_content" FATAL = "fatal" # auth / missing model / unrecognized 4xx -> fail the sample diff --git a/bench_core/migrate.py b/bench_core/migrate.py index f57b6dd..1138810 100644 --- a/bench_core/migrate.py +++ b/bench_core/migrate.py @@ -30,7 +30,9 @@ def _classify(name: str) -> str | None: if name.startswith("mmmupro_vision_"): return "mmmu_pro_vision" if name.startswith("mmmlulite_"): - return "mmmlu" + return "mmmlu_lite" + if name.startswith("mmmlufull_"): + return "mmmlu_full" if name.startswith("refcoco_"): return f"refcoco_{name.split('_')[1]}" if name.startswith("voxpopuli_aa_"): diff --git a/bench_core/reasoning.py b/bench_core/reasoning.py index 1e21b34..57ef362 100644 --- a/bench_core/reasoning.py +++ b/bench_core/reasoning.py @@ -13,10 +13,6 @@ @dataclass class ReasoningInjection: - """How to attach reasoning to a request. A given cap fills exactly one of - the mechanism fields; the rest stay empty/None so an adapter that reads the - wrong field simply sees "nothing to do".""" - kwargs: dict = field( default_factory=dict ) # top-level create() kwargs (reasoning_effort) @@ -32,8 +28,6 @@ class ReasoningInjection: def _resolve_value(mode: str, cap: ReasoningCap) -> Any: - """Concrete value for a mode: `off`/`on` come from the cap; an intermediate - mode passes through literally (caller owns vocabulary validity).""" if mode in _OFF_MODES: return cap.off_value if mode in _HIGH_MODES: @@ -42,9 +36,6 @@ def _resolve_value(mode: str, cap: ReasoningCap) -> Any: def _allow_temperature(mode: str, cap: ReasoningCap) -> bool: - """Temperature is allowed when thinking is not engaged, or when the host - tolerates temperature alongside active thinking. An off run that truly - disables thinking counts as "not engaged".""" if mode in _OFF_MODES and cap.true_off: return True return cap.temperature_when_on diff --git a/benchmarks/mmmlu/bench.py b/benchmarks/mmmlu/bench.py index db94369..4127da7 100644 --- a/benchmarks/mmmlu/bench.py +++ b/benchmarks/mmmlu/bench.py @@ -1,4 +1,9 @@ -"""MMMLU-lite: 14-language MCQ; per-language accuracy, macro-averaged headline.""" +"""MMMLU: 14-language MCQ; per-language accuracy, macro-averaged headline. + +Two variants (as in the original): `lite` (opencompass/mmmlu_lite, ~20k, the +default and what the archived leaderboard numbers use) and `full` +(openai/MMMLU, ~196k). Pick with --variant. +""" from __future__ import annotations @@ -10,9 +15,11 @@ NAME = "mmmlu" ID_KEY = "id" PRIMARY_METRIC = "macro_accuracy" +VARIANTS = ["lite", "full"] DEFAULTS = {"reasoning": "off", "rate_limit": 50, "max_in_flight": 8} _DATASET_LITE_ID = "opencompass/mmmlu_lite" +_DATASET_FULL_ID = "openai/MMMLU" _SPLIT = "test" LANGUAGES = [ "AR_XY", @@ -59,27 +66,43 @@ def parse_answer(text: str) -> str | None: return m.group(0) if m else None -def load_samples(sample_size: int | None = None) -> list[dict]: +def _build_sample(row: dict, lang: str, i: int, variant: str) -> dict: + # lite (opencompass) and full (openai/MMMLU) use different column names + + # id sources; the prompt/answer semantics are identical. + if variant == "lite": + return { + "id": f"{lang}:{i}", + "language": lang, + "subject": row["subject"], + "question": row["input"], + "a": row["A"], + "b": row["B"], + "c": row["C"], + "d": row["D"], + "answer": str(row["target"]).strip().upper(), + } + return { + "id": f"{lang}:{row.get('Unnamed: 0')}", + "language": lang, + "subject": row["Subject"], + "question": row["Question"], + "a": row["A"], + "b": row["B"], + "c": row["C"], + "d": row["D"], + "answer": str(row["Answer"]).strip().upper(), + } + + +def load_samples(sample_size: int | None = None, variant: str = "lite") -> list[dict]: from datasets import load_dataset + dataset_id = _DATASET_LITE_ID if variant == "lite" else _DATASET_FULL_ID samples = [] for lang in LANGUAGES: - ds = load_dataset(_DATASET_LITE_ID, lang, split=_SPLIT) + ds = load_dataset(dataset_id, lang, split=_SPLIT) for i, row in enumerate(ds): - row = dict(row) - samples.append( - { - "id": f"{lang}:{i}", - "language": lang, - "subject": row["subject"], - "question": row["input"], - "a": row["A"], - "b": row["B"], - "c": row["C"], - "d": row["D"], - "answer": str(row["target"]).strip().upper(), - } - ) + samples.append(_build_sample(dict(row), lang, i, variant)) return samples[:sample_size] if sample_size else samples diff --git a/benchmarks/obj_detection/bench.py b/benchmarks/obj_detection/bench.py index 0c8622d..37fa650 100644 --- a/benchmarks/obj_detection/bench.py +++ b/benchmarks/obj_detection/bench.py @@ -16,10 +16,26 @@ NAME = "refcoco" ID_KEY = "id" PRIMARY_METRIC = "accuracy" -VARIANTS = ["val", "testA", "testB", "test"] +# variant = split for base RefCOCO, or "plus-" / "g-" for +# RefCOCO+ / RefCOCOg (same underlying task, lmms-lab packaging). +VARIANTS = [ + "val", + "testA", + "testB", + "test", + "plus-val", + "plus-testA", + "plus-testB", + "g-val", + "g-test", +] DEFAULTS = {"reasoning": "off", "rate_limit": 25, "max_in_flight": 8} -_DATASET_ID = "lmms-lab/RefCOCO" +_DATASETS = { + "": "lmms-lab/RefCOCO", + "plus": "lmms-lab/RefCOCO+", + "g": "lmms-lab/RefCOCOg", +} _MAX_SIDE = 1024 IOU_THRESHOLD = 0.5 _THRESHOLDS = [0.3, 0.5, 0.7, 0.75, 0.9] @@ -190,10 +206,19 @@ def _sent_dims(w, h, max_side=_MAX_SIDE): return round(w * scale), round(h * scale) +def _parse_variant(variant: str) -> tuple[str, str]: + if variant.startswith("plus-"): + return _DATASETS["plus"], variant[len("plus-") :] + if variant.startswith("g-"): + return _DATASETS["g"], variant[len("g-") :] + return _DATASETS[""], variant + + def load_samples(sample_size: int | None = None, variant: str = "val") -> list[dict]: from datasets import load_dataset - ds = load_dataset(_DATASET_ID, split=variant) + dataset_id, split = _parse_variant(variant) + ds = load_dataset(dataset_id, split=split) rows = list(ds)[:sample_size] if sample_size else list(ds) samples = [] for i, row in enumerate(rows): diff --git a/scripts/report_scores.py b/scripts/report_scores.py index 8d28784..12546c5 100644 --- a/scripts/report_scores.py +++ b/scripts/report_scores.py @@ -200,8 +200,9 @@ def load_metrics() -> dict: "cer": _pct(d.get("corpus_cer")), "n": d.get("num_samples") or d.get("n"), } - elif benchmark == "mmmlu": - b["mmmlu"] = { + elif benchmark.startswith("mmmlu"): # mmmlu_lite / mmmlu_full + variant = benchmark[len("mmmlu_") :] if "_" in benchmark else "lite" + b[f"mmmlu_{variant}"] = { "macro": _pct(d.get("macro_accuracy")), "n": d.get("num_samples") or d.get("n"), "langs": { @@ -431,15 +432,18 @@ def get(model, key): print("\n=== GPQA Diamond ===") table([c for c, _ in [("Model", None)] + GPQA_COLS], rows, tsv) - # ---- MMMLU - rows = [] - for m in models: - d = get(m, "mmmlu") - if d: - rows.append([m, d["macro"]] + [d["langs"].get(lg) for lg in MMMLU_LANGS]) - if rows: - print("\n=== MMMLU ===") - table(["Model", "macro"] + MMMLU_LANGS, rows, tsv) + # ---- MMMLU (lite and/or full) + for variant in ("lite", "full"): + rows = [] + for m in models: + d = get(m, f"mmmlu_{variant}") + if d: + rows.append( + [m, d["macro"]] + [d["langs"].get(lg) for lg in MMMLU_LANGS] + ) + if rows: + print(f"\n=== MMMLU ({variant}) ===") + table(["Model", "macro"] + MMMLU_LANGS, rows, tsv) # ---- MMMU-Pro, subjects as rows in the sheet's order for setting, subjects in ( diff --git a/tests/test_mmmlu_bench.py b/tests/test_mmmlu_bench.py index 9e0fa61..d1723c7 100644 --- a/tests/test_mmmlu_bench.py +++ b/tests/test_mmmlu_bench.py @@ -22,6 +22,42 @@ def test_parse_answer(): assert bench.parse_answer("") is None +def test_build_sample_lite_and_full_variants(): + # lite (opencompass) schema + lite = bench._build_sample( + { + "subject": "math", + "input": "2+2?", + "A": "3", + "B": "4", + "C": "5", + "D": "6", + "target": "b", + }, + "EN", + 3, + "lite", + ) + assert lite["id"] == "EN:3" and lite["question"] == "2+2?" and lite["answer"] == "B" + # full (openai/MMMLU) schema — id from Unnamed: 0, Question/Subject/Answer columns + full = bench._build_sample( + { + "Unnamed: 0": 7, + "Subject": "math", + "Question": "2+2?", + "A": "3", + "B": "4", + "C": "5", + "D": "6", + "Answer": "b", + }, + "EN", + 0, + "full", + ) + assert full["id"] == "EN:7" and full["subject"] == "math" and full["answer"] == "B" + + def test_score_macro_averages_languages(): samples = [ {"id": "EN:0", "language": "EN", "subject": "math", "answer": "A"}, diff --git a/tests/test_refcoco_bench.py b/tests/test_refcoco_bench.py index b5ab977..c0032c1 100644 --- a/tests/test_refcoco_bench.py +++ b/tests/test_refcoco_bench.py @@ -20,6 +20,13 @@ ORACLE_BASE = "refcoco_val_fireworks_accounts_fireworks_models_inkling" +def test_parse_variant_restores_plus_and_g_datasets(): + assert bench._parse_variant("val") == ("lmms-lab/RefCOCO", "val") + assert bench._parse_variant("testA") == ("lmms-lab/RefCOCO", "testA") + assert bench._parse_variant("plus-testB") == ("lmms-lab/RefCOCO+", "testB") + assert bench._parse_variant("g-test") == ("lmms-lab/RefCOCOg", "test") + + def test_compute_iou(): assert bench.compute_iou([0, 0, 10, 10], [0, 0, 10, 10]) == 1.0 assert bench.compute_iou([0, 0, 10, 10], [20, 20, 30, 30]) == 0.0 From f6ccd7633fff4cf1e0873ae31a81da07616d7caa Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 16:55:36 +0530 Subject: [PATCH 21/38] chore: remove JigsawStack object-detection benchmark (unused) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deletes benchmarks/obj_detection/ob_det_api.py (the JigsawStack API benchmark) and refcoco.py (the shared helper lib that only ob_det_api still imported — the new bench.py is self-contained). Drops the JIGSAWSTACK_API_KEY from the README .env example and the ob_det_api run lines. obj_detection now = bench.py only. 174 tests pass. --- README.md | 8 +- bench_core/request.py | 5 - benchmarks/obj_detection/ob_det_api.py | 448 --------------------- benchmarks/obj_detection/refcoco.py | 536 ------------------------- 4 files changed, 2 insertions(+), 995 deletions(-) delete mode 100644 benchmarks/obj_detection/ob_det_api.py delete mode 100644 benchmarks/obj_detection/refcoco.py diff --git a/README.md b/README.md index 94983ac..12b9668 100644 --- a/README.md +++ b/README.md @@ -20,14 +20,13 @@ ANTHROPIC_API_KEY=... GEMINI_KEY=... OPENROUTER_API_KEY=... FIREWORKS_API_KEY=... -JIGSAWSTACK_API_KEY=... # only for benchmarks/obj_detection/ob_det_api.py ``` Benchmarks run through one CLI: `uv run python -m bench_core run --target --benchmark ` (see `python -m bench_core list-targets`). `--sample N` is a smoke test; re-running resumes from the per-run checkpoint (completed samples are -skipped). The remaining bespoke runners (`ob_det_api`, `olmocr_bench_reducto`, -BFCL) keep their own `--limit`/`--evaluate-only` flags. +skipped). The remaining bespoke runners (`olmocr_bench_reducto`, BFCL) keep their +own `--limit`/`--evaluate-only` flags. Two extra setup steps for specific benchmarks: @@ -179,9 +178,6 @@ uv run python -m bench_core run --target gpt-5.5 --benchmark refcoco -- uv run python -m bench_core run --target gemini-3.7-flash --benchmark refcoco --variant testA uv run python -m bench_core run --target inkling --benchmark refcoco --variant plus-testB uv run python -m bench_core run --target inkling --benchmark refcoco --variant g-val - -# JigsawStack object_detection API (a bespoke endpoint, not a chat model) -uv run -m benchmarks.obj_detection.ob_det_api ``` --- diff --git a/bench_core/request.py b/bench_core/request.py index 29ede2b..94d861a 100644 --- a/bench_core/request.py +++ b/bench_core/request.py @@ -12,11 +12,6 @@ class TextPart: @dataclass class ImagePart: - """Already-encoded image bytes + its mime (e.g. "image/jpeg" for OCRBench, - "image/png" for olmOCR — a *benchmark* choice; the wire *shape* is the - adapter's job). Build via `media.encode_image` so RGB-convert/resize happen - once, centrally, instead of in 1-of-9 variants.""" - data: bytes mime: str = "image/jpeg" diff --git a/benchmarks/obj_detection/ob_det_api.py b/benchmarks/obj_detection/ob_det_api.py deleted file mode 100644 index 778c358..0000000 --- a/benchmarks/obj_detection/ob_det_api.py +++ /dev/null @@ -1,448 +0,0 @@ -""" -RefCOCO benchmark for the JigsawStack /v1/object_detection API. - -Mirrors the protocol used by `refcoco.py` (interfaze-beta) so numbers are -directly comparable on the same splits at Acc@IoU=0.5. The only thing that -changes is the inference call: instead of asking a VLM to emit a `Box: [...]` -line, we hit JigsawStack's object_detection endpoint with the referring -expression as a prompt and parse `objects[0].bounds`. - -Per-sample we record both: - - acc / iou — IoU of the FIRST returned object vs. the GT box. - This is the honest, deployable number. - - oracle_iou / oracle_correct — best IoU across ALL returned objects. - Diagnostic upper bound — tells us whether the - right box was in the response and we just picked - wrong. - -Datasets (lmms-lab packaging — same data as standard RefCOCO splits): - lmms-lab/RefCOCO val 8811, test 5000, testA 1975, testB 1810 - lmms-lab/RefCOCO+ val 8823, testA 1975, testB 1798 - lmms-lab/RefCOCOg val 7573, test 9602 (UMD split) - -Usage: - uv run -m benchmarks.obj_detection.ob_det_api # RefCOCO val - uv run -m benchmarks.obj_detection.ob_det_api --split testA - uv run -m benchmarks.obj_detection.ob_det_api --dataset lmms-lab/RefCOCO+ --split testB - uv run -m benchmarks.obj_detection.ob_det_api --limit 20 - uv run -m benchmarks.obj_detection.ob_det_api --evaluate-only - -Env: JIGSAWSTACK_API_KEY must be set (loaded from .env). - -Checkpointing: results/jigsawstack___responses.jsonl — -each successful sample is appended as it finishes; reruns skip completed ids. -""" - -import sys -import json -import os -import time -import base64 -import asyncio -import argparse -import traceback -import concurrent.futures -from io import BytesIO -from pathlib import Path - -import httpx -from datasets import load_dataset -from dotenv import load_dotenv -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -# Reuse sample construction, IoU, and resume logic from refcoco.py — keeps -# scoring byte-for-byte identical to the interfaze run. -from benchmarks.obj_detection.refcoco import ( # noqa: E402 - JsonlWriter, - build_samples, - coco_bbox_to_xyxy, - compute_iou, - load_completed_ids, - load_records, - pil_to_data_url, -) - -load_dotenv() - -RESULTS_DIR = PROJECT_ROOT / "results" -DEFAULT_DATASET = "lmms-lab/RefCOCO" -DEFAULT_SPLIT = "val" -CONCURRENCY = 10 # JigsawStack rate-limits more aggressively than interfaze -MAX_RETRIES = 3 -IOU_THRESHOLD = 0.5 -REQUEST_TIMEOUT = 120.0 - -JIGSAWSTACK_URL = "http://localhost:3000/api/v1/object_detection" -JIGSAWSTACK_API_KEY = os.getenv("JIGSAWSTACK_API_KEY") -if not JIGSAWSTACK_API_KEY: - raise RuntimeError( - "JIGSAWSTACK_API_KEY is not set. Add it to .env " - "(get one from https://jigsawstack.com/dashboard)." - ) - - -def invoke_jigsawstack_obj_detection(image_data_url: str, expression: str) -> dict: - """POST to /v1/object_detection with the referring expression as a prompt. - - `annotated_image=False` skips the rendered overlay we don't need, which - saves both bandwidth and server-side time. `x-jigsaw-skip-cache: true` - matches the example curl and ensures we measure real inference latency - rather than cache hits.""" - payload = { - "url": image_data_url, - "annotated_image": False, - "features": ["object"], - "prompts": [expression], - } - headers = { - "x-api-key": JIGSAWSTACK_API_KEY, - "x-jigsaw-skip-cache": "true", - "Content-Type": "application/json", - } - with httpx.Client(timeout=REQUEST_TIMEOUT) as client: - resp = client.post(JIGSAWSTACK_URL, json=payload, headers=headers) - if resp.status_code != 200: - raise RuntimeError(f"HTTP {resp.status_code}: {resp.text[:500]}") - data = resp.json() - if not data.get("success", False): - raise RuntimeError(f"API error: {str(data)[:500]}") - return data - - -def bounds_to_xyxy(bounds: dict) -> list[float] | None: - """Convert JigsawStack's `bounds` object to [x1, y1, x2, y2]. - - The API returns four corners (top_left/top_right/bottom_left/bottom_right); - we only need the diagonal pair. Falls back to width/height if a corner is - missing.""" - tl = bounds.get("top_left") or {} - br = bounds.get("bottom_right") or {} - if "x" in tl and "y" in tl and "x" in br and "y" in br: - return [float(tl["x"]), float(tl["y"]), float(br["x"]), float(br["y"])] - if "x" in tl and "y" in tl and "width" in bounds and "height" in bounds: - x1, y1 = float(tl["x"]), float(tl["y"]) - return [x1, y1, x1 + float(bounds["width"]), y1 + float(bounds["height"])] - return None - - -def extract_boxes(api_response: dict) -> list[list[float]]: - """Return all detected boxes as [x1,y1,x2,y2], in the order returned.""" - boxes: list[list[float]] = [] - for obj in api_response.get("objects") or []: - bounds = obj.get("bounds") - if not bounds: - continue - xyxy = bounds_to_xyxy(bounds) - if xyxy is not None: - boxes.append(xyxy) - return boxes - - -async def process_sample( - sample: dict, semaphore: asyncio.Semaphore, writer: JsonlWriter, progress: dict -) -> dict | None: - orig_w, orig_h = sample["image"].size - data_url, sent_w, sent_h = pil_to_data_url(sample["image"]) - sx = sent_w / orig_w - sy = sent_h / orig_h - gt_xyxy = coco_bbox_to_xyxy(sample["bbox_xywh"], sx, sy) - - last_error: str | None = None - - for attempt in range(1, MAX_RETRIES + 1): - start = time.perf_counter() - try: - async with semaphore: - start = time.perf_counter() - api_response = await asyncio.to_thread( - invoke_jigsawstack_obj_detection, - data_url, - sample["expression"], - ) - latency_ms = int((time.perf_counter() - start) * 1000) - - boxes = extract_boxes(api_response) - num_objects = len(boxes) - - # Honest pick: first object, matching how a downstream caller - # would use the API without ground truth. - pred_box = boxes[0] if boxes else None - iou = compute_iou(pred_box, gt_xyxy) if pred_box else 0.0 - correct = pred_box is not None and iou >= IOU_THRESHOLD - - # Oracle: best IoU across all returned boxes — tells us how often - # the right answer was in the response but we picked wrong. - if boxes: - oracle_iou = max(compute_iou(b, gt_xyxy) for b in boxes) - else: - oracle_iou = 0.0 - oracle_correct = oracle_iou >= IOU_THRESHOLD - - log_id = api_response.get("log_id") - usage = api_response.get("_usage") - - record = { - "id": sample["id"], - "question_id": sample["question_id"], - "file_name": sample["file_name"], - "expression": sample["expression"], - "all_expressions": sample["all_expressions"], - "image_width": sent_w, - "image_height": sent_h, - "gt_bbox_xyxy": gt_xyxy, - "pred_bbox_xyxy": pred_box, - "all_pred_boxes": boxes, - "num_objects": num_objects, - "iou": iou, - "correct": correct, - "oracle_iou": oracle_iou, - "oracle_correct": oracle_correct, - "response": api_response, - "log_id": log_id, - "usage": usage, - "latency_ms": latency_ms, - "attempts": attempt, - } - await writer.append(record) - - progress["done"] += 1 - if correct: - progress["correct"] += 1 - mark = "OK" - elif oracle_correct: - mark = "o " # right box was returned, just not first - else: - mark = "X " - tqdm.write( - f"[{progress['done']}/{progress['total']}] {mark} " - f"id={sample['id']} iou={iou:.3f} oracle={oracle_iou:.3f} " - f"n={num_objects} pred={pred_box} " - f"gt={[round(x, 1) for x in gt_xyxy]} " - f"latency={latency_ms}ms log_id={log_id} attempt={attempt}" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - tqdm.write( - f"[error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(2 ** (attempt - 1)) - - progress["failed"] += 1 - tqdm.write(f"[FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") - return None - - -def compute_metrics(results: list[dict]) -> dict: - total = len(results) - correct = sum(1 for r in results if r.get("correct")) - oracle_correct = sum(1 for r in results if r.get("oracle_correct")) - no_objects = sum(1 for r in results if not r.get("num_objects")) - ious = [r["iou"] for r in results if isinstance(r.get("iou"), (int, float))] - oracle_ious = [ - r["oracle_iou"] - for r in results - if isinstance(r.get("oracle_iou"), (int, float)) - ] - latencies = [ - r["latency_ms"] for r in results if isinstance(r.get("latency_ms"), int) - ] - n_objects = [ - r["num_objects"] for r in results if isinstance(r.get("num_objects"), int) - ] - - thresholds = [0.3, 0.5, 0.7, 0.75, 0.9] - at_threshold = { - f"acc@{t}": (sum(1 for iou in ious if iou >= t) / len(ious) if ious else 0) - for t in thresholds - } - oracle_at_threshold = { - f"oracle_acc@{t}": ( - sum(1 for iou in oracle_ious if iou >= t) / len(oracle_ious) - if oracle_ious - else 0 - ) - for t in thresholds - } - - latency_stats = {} - if latencies: - lats = sorted(latencies) - n = len(lats) - latency_stats = { - "count": n, - "mean_ms": sum(lats) / n, - "p50_ms": lats[n // 2], - "p90_ms": lats[min(n - 1, int(n * 0.9))], - "p99_ms": lats[min(n - 1, int(n * 0.99))], - "max_ms": lats[-1], - } - - return { - "accuracy": correct / total if total else 0.0, - "oracle_accuracy": oracle_correct / total if total else 0.0, - "correct": correct, - "oracle_correct": oracle_correct, - "total": total, - "no_objects_returned": no_objects, - "mean_iou": sum(ious) / len(ious) if ious else 0.0, - "mean_oracle_iou": sum(oracle_ious) / len(oracle_ious) if oracle_ious else 0.0, - "mean_objects_per_response": sum(n_objects) / len(n_objects) - if n_objects - else 0.0, - "iou_thresholds": at_threshold, - "oracle_iou_thresholds": oracle_at_threshold, - "latency": latency_stats, - } - - -def print_summary(metrics: dict, dataset_name: str, split: str): - print(f"\n{'=' * 60}") - print( - f"Grounding Results — {dataset_name} / {split} (JigsawStack object_detection)" - ) - print(f"{'=' * 60}") - print( - f"Acc@IoU=0.5 : {metrics['accuracy']:.4f} ({metrics['correct']}/{metrics['total']})" - ) - print( - f"Oracle Acc@IoU=0.5 : {metrics['oracle_accuracy']:.4f} ({metrics['oracle_correct']}/{metrics['total']})" - ) - print(f"Mean IoU : {metrics['mean_iou']:.4f}") - print(f"Mean Oracle IoU : {metrics['mean_oracle_iou']:.4f}") - print(f"Avg objects / resp : {metrics['mean_objects_per_response']:.2f}") - print(f"No-object responses: {metrics['no_objects_returned']}") - if metrics.get("latency"): - lat = metrics["latency"] - print( - f"Latency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " - f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms max={lat['max_ms']}ms" - ) - print("\nIoU thresholds (first-object pick):") - for t, acc in metrics["iou_thresholds"].items(): - print(f" {t}: {acc:.4f}") - print("\nIoU thresholds (oracle / best-of-N):") - for t, acc in metrics["oracle_iou_thresholds"].items(): - print(f" {t}: {acc:.4f}") - - -def build_tag(dataset: str, split: str) -> str: - ds_slug = dataset.split("/")[-1].lower().replace("+", "plus") - return f"jigsawstack_{ds_slug}_{split}" - - -async def run_predictions( - dataset_name: str, split: str, pred_path: Path, limit: int | None -): - print(f"Loading {dataset_name}, split={split}...") - dataset = load_dataset(dataset_name, split=split) - print(f"Loaded {len(dataset)} rows") - - samples = build_samples(dataset) - done_ids = load_completed_ids(pred_path) - pending = [s for s in samples if s["id"] not in done_ids] - if limit is not None: - pending = pending[:limit] - print(f"--limit applied: will run at most {limit} sample(s)") - print( - f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})" - ) - if not pending: - return - - asyncio.get_running_loop().set_default_executor( - concurrent.futures.ThreadPoolExecutor(max_workers=CONCURRENCY) - ) - - writer = JsonlWriter(pred_path) - semaphore = asyncio.Semaphore(CONCURRENCY) - progress = {"total": len(pending), "done": 0, "correct": 0, "failed": 0} - tasks = [process_sample(s, semaphore, writer, progress) for s in pending] - try: - await tqdm_asyncio.gather(*tasks, desc=f"{dataset_name}/{split}") - except Exception: - traceback.print_exc() - print( - f"\nRun finished: {progress['done']}/{progress['total']} answered, " - f"{progress['correct']} correct, {progress['failed']} failed." - ) - - -def run_evaluation(dataset_name: str, split: str, pred_path: Path, metrics_path: Path): - if not pred_path.exists(): - print(f"No predictions found at {pred_path}") - sys.exit(1) - results = load_records(pred_path) - if not results: - print(f"No records in {pred_path}") - sys.exit(1) - - metrics = compute_metrics(results) - print_summary(metrics, dataset_name, split) - output = { - **metrics, - "dataset": dataset_name, - "split": split, - "concurrency": CONCURRENCY, - "iou_threshold": IOU_THRESHOLD, - "model": "jigsawstack:object_detection", - } - metrics_path.parent.mkdir(parents=True, exist_ok=True) - with open(metrics_path, "w") as f: - json.dump(output, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - parser = argparse.ArgumentParser( - description="RefCOCO / RefCOCO+ / RefCOCOg benchmark for the JigsawStack " - "object_detection API (Acc@IoU=0.5, comparable to refcoco.py)" - ) - parser.add_argument( - "--dataset", - default=DEFAULT_DATASET, - help="lmms-lab/RefCOCO | lmms-lab/RefCOCO+ | lmms-lab/RefCOCOg", - ) - parser.add_argument( - "--split", - default=DEFAULT_SPLIT, - help="val | testA | testB | test (availability varies by dataset)", - ) - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument( - "--limit", - type=int, - default=None, - help="Only run the first N unanswered samples", - ) - args = parser.parse_args() - - tag = build_tag(args.dataset, args.split) - pred_path = RESULTS_DIR / f"{tag}_responses.jsonl" - metrics_path = RESULTS_DIR / f"{tag}_metrics.json" - - if args.evaluate_only: - run_evaluation(args.dataset, args.split, pred_path, metrics_path) - elif args.predict_only: - asyncio.run( - run_predictions(args.dataset, args.split, pred_path, limit=args.limit) - ) - else: - asyncio.run( - run_predictions(args.dataset, args.split, pred_path, limit=args.limit) - ) - run_evaluation(args.dataset, args.split, pred_path, metrics_path) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/obj_detection/refcoco.py b/benchmarks/obj_detection/refcoco.py deleted file mode 100644 index ecd39c3..0000000 --- a/benchmarks/obj_detection/refcoco.py +++ /dev/null @@ -1,536 +0,0 @@ -""" -RefCOCO (+ optionally RefCOCO+ / RefCOCOg) benchmark for Interfaze. - -This is the canonical VLM grounding benchmark: given an image and a referring -expression, the model outputs a bounding box. A prediction is considered -correct if its IoU with the ground-truth box > 0.5 (Acc@IoU=0.5). This is -exactly the metric reported by Qwen3-VL, DeepSeek-VL2, InternVL 2.5/3, -GLM-4.x-V, CogVLM-Grounding, Kosmos-2 and essentially every VLM paper. - -Datasets (all lmms-lab packaging — same underlying RefCOCO data as the -original Kazemzadeh et al. 2014 / Mao et al. 2016 splits): - lmms-lab/RefCOCO val 8811, test 5000, testA 1975, testB 1810 - lmms-lab/RefCOCO+ val 8823, testA 1975, testB 1798 - lmms-lab/RefCOCOg val 7573, test 9602 (UMD split) - -Usage: - uv run -m benchmarks.obj_detection.refcoco # RefCOCO val - uv run -m benchmarks.obj_detection.refcoco --split testA - uv run -m benchmarks.obj_detection.refcoco --dataset lmms-lab/RefCOCO+ --split testB - uv run -m benchmarks.obj_detection.refcoco --limit 20 - uv run -m benchmarks.obj_detection.refcoco --evaluate-only - -Checkpointing: - Each successful sample is appended to results/_responses.jsonl - as it finishes. Reruns only query samples still missing. Failed-after- - retries samples are NOT written, so they retry on next run. -""" - -import sys -import json -import os -import re -import time -import base64 -import asyncio -import argparse -import traceback -import concurrent.futures -from io import BytesIO -from pathlib import Path - -from datasets import load_dataset -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -from src.commons import invoke_interfaze # noqa: E402 - -RESULTS_DIR = PROJECT_ROOT / "results" -DEFAULT_DATASET = "lmms-lab/RefCOCO" -DEFAULT_SPLIT = "val" -REASONING_EFFORT = None # None => omit the reasoning_effort param entirely (interfaze default = off) -TEMPERATURE = 0.0 # deterministic decoding — matches standard VLM grounding eval protocol -CONCURRENCY = 25 -MAX_RETRIES = 3 -IOU_THRESHOLD = 0.5 - -# Canonical RefCOCO grounding prompt — wording from lmms-eval's `refcoco_rec` -# task (InternVL / Qwen-VL convention) plus the single output-format line that -# published "general VLM on RefCOCO" tables (LLaVA, GPT-4V, Claude) use to -# disambiguate xyxy vs yxyx ordering. No coord-space pinning, no step-by-step. -# `{width}` and `{height}` are accepted but unused so callers don't break. -PROMPT_TEMPLATE = ( - "Please provide the bounding box coordinate of the region this sentence describes: " - "{expression}\n\n" - "Output the coordinates in the format [x_min, y_min, x_max, y_max]." -) - - -BOX_LINE_PATTERN = re.compile( - r"(?im)^\s*(?:box|answer|bounding\s*box)[\s:]*" - r"\[?\s*(-?\d+(?:\.\d+)?)\s*[,\s]\s*(-?\d+(?:\.\d+)?)\s*[,\s]\s*" - r"(-?\d+(?:\.\d+)?)\s*[,\s]\s*(-?\d+(?:\.\d+)?)\s*\]?" -) -BOXED_PATTERN = re.compile( - r"\\boxed\{\s*\[?\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*" - r"(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\]?\s*\}" -) -BARE_4TUPLE = re.compile( - r"\[\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*" - r"(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\]" -) - - -class JsonlWriter: - def __init__(self, path: Path): - self.path = path - self.path.parent.mkdir(parents=True, exist_ok=True) - self._lock = asyncio.Lock() - - async def append(self, record: dict): - line = json.dumps(record, ensure_ascii=False) - async with self._lock: - with open(self.path, "a", encoding="utf-8") as f: - f.write(line + "\n") - f.flush() - os.fsync(f.fileno()) - - -def pil_to_data_url(image, max_side: int = 1024) -> tuple[str, int, int]: - """Encode PIL image as data URL. Returns (url, width, height) of the - image as sent to the model — coords are interpreted in this space.""" - if image.mode != "RGB": - image = image.convert("RGB") - w, h = image.size - scale = min(1.0, max_side / max(w, h)) - if scale < 1.0: - new_w, new_h = int(round(w * scale)), int(round(h * scale)) - image = image.resize((new_w, new_h)) - w, h = new_w, new_h - buf = BytesIO() - image.save(buf, format="JPEG", quality=92) - b64 = base64.b64encode(buf.getvalue()).decode("utf-8") - return f"data:image/jpeg;base64,{b64}", w, h - - -def coco_bbox_to_xyxy(bbox, scale_x: float = 1.0, scale_y: float = 1.0) -> list[float]: - """RefCOCO ground truth is COCO format [x, y, w, h]. Convert to - [x1, y1, x2, y2] and optionally rescale into a resized image space.""" - x, y, w, h = bbox - return [x * scale_x, y * scale_y, (x + w) * scale_x, (y + h) * scale_y] - - -JSON_TLBR_PATTERN = re.compile( - r'"top_left"\s*:\s*\{[^}]*?"x"\s*:\s*(-?\d+(?:\.\d+)?)[^}]*?"y"\s*:\s*(-?\d+(?:\.\d+)?)' - r'[^}]*?\}[^}]*?"bottom_right"\s*:\s*\{[^}]*?"x"\s*:\s*(-?\d+(?:\.\d+)?)' - r'[^}]*?"y"\s*:\s*(-?\d+(?:\.\d+)?)', - re.DOTALL, -) -JSON_BOX2D_PATTERN = re.compile( - r'"box_2d"\s*:\s*\[\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*' - r'(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\]' -) - - -def parse_box(text: str, width: int, height: int) -> list[float] | None: - """Extract a [x1, y1, x2, y2] bounding box in pixel space from the model - response. Robust to the formats general VLMs commonly emit on RefCOCO: - - - "Box: [a,b,c,d]" / "\\boxed{[a,b,c,d]}" → [x1,y1,x2,y2] - - bare last [a,b,c,d] in tail → [x1,y1,x2,y2] - - {"top_left":{"x","y"}, "bottom_right":{"x","y"}} - - {"box_2d":[ymin,xmin,ymax,xmax]} (Gemini convention, 0-1000) - - Coordinate space is auto-detected: 0-1.0 floats, 0-1000 normalized, or - raw pixels — picks whichever fits the image dimensions. - """ - if not text: - return None - - # 1. JSON object with top_left / bottom_right corners — already xyxy. - m = None - for match in JSON_TLBR_PATTERN.finditer(text): - m = match - if m is not None: - return _normalize_to_pixels( - [float(m.group(i)) for i in range(1, 5)], width, height, order="xyxy" - ) - - # 2. Gemini-style {"box_2d": [...]} — [ymin, xmin, ymax, xmax]. - m = None - for match in JSON_BOX2D_PATTERN.finditer(text): - m = match - if m is not None: - return _normalize_to_pixels( - [float(m.group(i)) for i in range(1, 5)], width, height, order="yxyx" - ) - - # 3. Box: / Answer: / \boxed{} prefixed line — assume xyxy. - for pattern in (BOX_LINE_PATTERN, BOXED_PATTERN): - matches = list(pattern.finditer(text)) - if matches: - m = matches[-1] - return _normalize_to_pixels( - [float(m.group(i)) for i in range(1, 5)], width, height, order="xyxy" - ) - - # 4. Fallback — last bare 4-tuple in the tail. - tail = text[-800:] - matches = list(BARE_4TUPLE.finditer(tail)) - if matches: - m = matches[-1] - return _normalize_to_pixels( - [float(m.group(i)) for i in range(1, 5)], width, height, order="xyxy" - ) - - return None - - -def _normalize_to_pixels( - box: list[float], width: int, height: int, order: str -) -> list[float]: - """Convert a 4-tuple to pixel-space [x1, y1, x2, y2] given its order - ('xyxy' or 'yxyx') and auto-detected coordinate scale (0-1.0 floats / - 0-1000 normalized / raw pixels). - """ - max_coord = max(abs(c) for c in box) - # Scale detection: pick the smallest scale whose interpretation fits - # the image, preferring 0-1 then 0-1000 then pixel. - if max_coord <= 1.0: - sx, sy = float(width), float(height) - elif max_coord <= 1000 and ( - max(width, height) > 1000 - or box[0] > width or box[2] > width - or box[1] > height or box[3] > height - ): - sx, sy = width / 1000.0, height / 1000.0 - else: - sx = sy = 1.0 - - if order == "xyxy": - return [box[0] * sx, box[1] * sy, box[2] * sx, box[3] * sy] - # yxyx (Gemini box_2d) → swap to xyxy and apply per-axis scale - return [box[1] * sx, box[0] * sy, box[3] * sx, box[2] * sy] - - -def compute_iou(a: list[float], b: list[float]) -> float: - ax1, ay1, ax2, ay2 = a - bx1, by1, bx2, by2 = b - # Normalize in case of (x1,y1,x2,y2) with swapped corners. - ax1, ax2 = min(ax1, ax2), max(ax1, ax2) - ay1, ay2 = min(ay1, ay2), max(ay1, ay2) - bx1, bx2 = min(bx1, bx2), max(bx1, bx2) - by1, by2 = min(by1, by2), max(by1, by2) - ix1 = max(ax1, bx1) - iy1 = max(ay1, by1) - ix2 = min(ax2, bx2) - iy2 = min(ay2, by2) - if ix1 >= ix2 or iy1 >= iy2: - return 0.0 - inter = (ix2 - ix1) * (iy2 - iy1) - area_a = max(0.0, (ax2 - ax1)) * max(0.0, (ay2 - ay1)) - area_b = max(0.0, (bx2 - bx1)) * max(0.0, (by2 - by1)) - union = area_a + area_b - inter - return inter / union if union > 0 else 0.0 - - -def build_samples(dataset) -> list[dict]: - """One sample per row, using the first referring expression. Matches - the convention used by lmms-eval / InternVL grounding evaluation.""" - samples = [] - for i, row in enumerate(dataset): - answers = row.get("answer") - if isinstance(answers, str): - expressions = [answers] - elif isinstance(answers, list) and answers: - expressions = [str(a) for a in answers if str(a).strip()] - else: - expressions = [] - if not expressions: - continue - samples.append({ - "id": f"{row.get('question_id', i)}_{i}", - "question_id": row.get("question_id"), - "file_name": row.get("file_name"), - "image": row["image"], - "expression": expressions[0], - "all_expressions": expressions, - "bbox_xywh": list(row["bbox"]), - }) - return samples - - -def load_completed_ids(path: Path) -> set[str]: - if not path.exists(): - return set() - done: set[str] = set() - with open(path, encoding="utf-8") as f: - for line_no, line in enumerate(f, 1): - line = line.strip() - if not line: - continue - try: - rec = json.loads(line) - except json.JSONDecodeError: - tqdm.write(f"[resume] skipping malformed line {line_no} in {path}") - continue - if rec.get("response"): - done.add(str(rec["id"])) - return done - - -def load_records(path: Path) -> list[dict]: - records: list[dict] = [] - if not path.exists(): - return records - with open(path, encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - records.append(json.loads(line)) - except json.JSONDecodeError: - continue - by_id: dict[str, dict] = {} - for r in records: - by_id[str(r["id"])] = r - return list(by_id.values()) - - -async def process_sample(sample: dict, semaphore: asyncio.Semaphore, - writer: JsonlWriter, progress: dict) -> dict | None: - orig_w, orig_h = sample["image"].size - data_url, sent_w, sent_h = pil_to_data_url(sample["image"]) - sx = sent_w / orig_w - sy = sent_h / orig_h - gt_xyxy = coco_bbox_to_xyxy(sample["bbox_xywh"], sx, sy) - - prompt = PROMPT_TEMPLATE.format( - width=sent_w, height=sent_h, expression=sample["expression"] - ) - messages = [{ - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - {"type": "image_url", "image_url": {"url": data_url}}, - ], - }] - last_error: str | None = None - - for attempt in range(1, MAX_RETRIES + 1): - start = time.perf_counter() - try: - async with semaphore: - start = time.perf_counter() - response = await asyncio.to_thread( - invoke_interfaze, - messages, - reasoning_effort=REASONING_EFFORT, - temperature=TEMPERATURE, - ) - latency_ms = int((time.perf_counter() - start) * 1000) - content = (response.choices[0].message.content or "").strip() - request_id = getattr(response, "id", None) - if not content: - last_error = "empty response content" - raise RuntimeError(last_error) - - pred_box = parse_box(content, sent_w, sent_h) - iou = compute_iou(pred_box, gt_xyxy) if pred_box else 0.0 - correct = pred_box is not None and iou >= IOU_THRESHOLD - - record = { - "id": sample["id"], - "question_id": sample["question_id"], - "file_name": sample["file_name"], - "expression": sample["expression"], - "all_expressions": sample["all_expressions"], - "image_width": sent_w, - "image_height": sent_h, - "gt_bbox_xyxy": gt_xyxy, - "pred_bbox_xyxy": pred_box, - "iou": iou, - "correct": correct, - "response": content, - "request_id": request_id, - "latency_ms": latency_ms, - "attempts": attempt, - } - await writer.append(record) - - progress["done"] += 1 - if correct: - progress["correct"] += 1 - mark = "OK" - else: - mark = "X " - tqdm.write( - f"[{progress['done']}/{progress['total']}] {mark} " - f"id={sample['id']} iou={iou:.3f} " - f"pred={pred_box} gt={[round(x,1) for x in gt_xyxy]} " - f"latency={latency_ms}ms req_id={request_id} attempt={attempt}" - ) - return record - - except Exception as e: - latency_ms = int((time.perf_counter() - start) * 1000) - last_error = f"{type(e).__name__}: {e}" - tqdm.write( - f"[error] id={sample['id']} attempt={attempt}/{MAX_RETRIES} " - f"latency={latency_ms}ms error={last_error}" - ) - if attempt < MAX_RETRIES: - await asyncio.sleep(2 ** (attempt - 1)) - - progress["failed"] += 1 - tqdm.write(f"[FAILED] id={sample['id']} after {MAX_RETRIES} attempts: {last_error}") - return None - - -def compute_metrics(results: list[dict]) -> dict: - total = len(results) - correct = sum(1 for r in results if r.get("correct")) - unparsed = sum(1 for r in results if r.get("pred_bbox_xyxy") is None) - ious = [r["iou"] for r in results if isinstance(r.get("iou"), (int, float))] - latencies = [r["latency_ms"] for r in results if isinstance(r.get("latency_ms"), int)] - - # Threshold sweep for sanity - thresholds = [0.3, 0.5, 0.7, 0.75, 0.9] - at_threshold = {f"acc@{t}": sum(1 for iou in ious if iou >= t) / len(ious) if ious else 0 - for t in thresholds} - - latency_stats = {} - if latencies: - lats = sorted(latencies) - n = len(lats) - latency_stats = { - "count": n, "mean_ms": sum(lats) / n, - "p50_ms": lats[n // 2], - "p90_ms": lats[min(n - 1, int(n * 0.9))], - "p99_ms": lats[min(n - 1, int(n * 0.99))], - "max_ms": lats[-1], - } - - return { - "accuracy": correct / total if total else 0.0, - "correct": correct, - "total": total, - "unparsed": unparsed, - "mean_iou": sum(ious) / len(ious) if ious else 0.0, - "iou_thresholds": at_threshold, - "latency": latency_stats, - } - - -def print_summary(metrics: dict, dataset_name: str, split: str): - print(f"\n{'=' * 60}") - print(f"Grounding Results — {dataset_name} / {split} (Interfaze, reasoning={REASONING_EFFORT})") - print(f"{'=' * 60}") - print(f"Acc@IoU=0.5 : {metrics['accuracy']:.4f} ({metrics['correct']}/{metrics['total']})") - print(f"Mean IoU : {metrics['mean_iou']:.4f}") - print(f"Unparsed : {metrics['unparsed']}") - if metrics.get("latency"): - lat = metrics["latency"] - print(f"Latency : mean={lat['mean_ms']:.0f}ms p50={lat['p50_ms']}ms " - f"p90={lat['p90_ms']}ms p99={lat['p99_ms']}ms max={lat['max_ms']}ms") - print("\nIoU thresholds:") - for t, acc in metrics["iou_thresholds"].items(): - print(f" {t}: {acc:.4f}") - - -def build_tag(dataset: str, split: str) -> str: - ds_slug = dataset.split("/")[-1].lower().replace("+", "plus") - return f"{ds_slug}_{split}" - - -async def run_predictions(dataset_name: str, split: str, pred_path: Path, limit: int | None): - print(f"Loading {dataset_name}, split={split}...") - dataset = load_dataset(dataset_name, split=split) - print(f"Loaded {len(dataset)} rows") - - samples = build_samples(dataset) - done_ids = load_completed_ids(pred_path) - pending = [s for s in samples if s["id"] not in done_ids] - if limit is not None: - pending = pending[:limit] - print(f"--limit applied: will run at most {limit} sample(s)") - print(f"Resume: {len(done_ids)} already completed, {len(pending)} remaining " - f"(checkpoint: {pred_path})") - if not pending: - return - - # Default asyncio thread pool is min(32, cpu_count + 4) — typically 12-16 - # on macOS, which would cap real in-flight HTTP calls below CONCURRENCY. - asyncio.get_running_loop().set_default_executor( - concurrent.futures.ThreadPoolExecutor(max_workers=CONCURRENCY) - ) - - writer = JsonlWriter(pred_path) - semaphore = asyncio.Semaphore(CONCURRENCY) - progress = {"total": len(pending), "done": 0, "correct": 0, "failed": 0} - tasks = [process_sample(s, semaphore, writer, progress) for s in pending] - try: - await tqdm_asyncio.gather(*tasks, desc=f"{dataset_name}/{split}") - except Exception: - traceback.print_exc() - print(f"\nRun finished: {progress['done']}/{progress['total']} answered, " - f"{progress['correct']} correct, {progress['failed']} failed.") - - -def run_evaluation(dataset_name: str, split: str, pred_path: Path, metrics_path: Path): - if not pred_path.exists(): - print(f"No predictions found at {pred_path}") - sys.exit(1) - results = load_records(pred_path) - if not results: - print(f"No records in {pred_path}") - sys.exit(1) - - metrics = compute_metrics(results) - print_summary(metrics, dataset_name, split) - output = { - **metrics, - "dataset": dataset_name, "split": split, - "reasoning_effort": REASONING_EFFORT, "concurrency": CONCURRENCY, - "iou_threshold": IOU_THRESHOLD, "model": "interfaze-beta", - } - metrics_path.parent.mkdir(parents=True, exist_ok=True) - with open(metrics_path, "w") as f: - json.dump(output, f, indent=2) - print(f"\nMetrics saved to {metrics_path}") - - -def main(): - parser = argparse.ArgumentParser( - description="RefCOCO / RefCOCO+ / RefCOCOg benchmark for Interfaze " - "(Acc@IoU=0.5, comparable to Qwen3-VL / DeepSeek-VL2 / InternVL / GLM-V)" - ) - parser.add_argument("--dataset", default=DEFAULT_DATASET, - help="lmms-lab/RefCOCO | lmms-lab/RefCOCO+ | lmms-lab/RefCOCOg") - parser.add_argument("--split", default=DEFAULT_SPLIT, - help="val | testA | testB | test (availability varies by dataset)") - parser.add_argument("--predict-only", action="store_true") - parser.add_argument("--evaluate-only", action="store_true") - parser.add_argument("--limit", type=int, default=None, - help="Only run the first N unanswered samples") - args = parser.parse_args() - - tag = build_tag(args.dataset, args.split) - pred_path = RESULTS_DIR / f"{tag}_responses.jsonl" - metrics_path = RESULTS_DIR / f"{tag}_metrics.json" - - if args.evaluate_only: - run_evaluation(args.dataset, args.split, pred_path, metrics_path) - elif args.predict_only: - asyncio.run(run_predictions(args.dataset, args.split, pred_path, limit=args.limit)) - else: - asyncio.run(run_predictions(args.dataset, args.split, pred_path, limit=args.limit)) - run_evaluation(args.dataset, args.split, pred_path, metrics_path) - - -if __name__ == "__main__": - main() From 2edfd5c2ae3bfc297dcff01c66e1a7bf015330e1 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 17:02:01 +0530 Subject: [PATCH 22/38] docs: reflow README setup section --- README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/README.md b/README.md index 12b9668..6387d33 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,7 @@ OPENROUTER_API_KEY=... FIREWORKS_API_KEY=... ``` -Benchmarks run through one CLI: `uv run python -m bench_core run --target ---benchmark ` (see `python -m bench_core list-targets`). `--sample N` is a -smoke test; re-running resumes from the per-run checkpoint (completed samples are -skipped). The remaining bespoke runners (`olmocr_bench_reducto`, BFCL) keep their -own `--limit`/`--evaluate-only` flags. +Benchmarks run through one CLI: `uv run python -m bench_core run --target --benchmark ` (see `python -m bench_core list-targets`). `--sample N` is a smoke test; re-running resumes from the per-run checkpoint (completed samples are skipped). The remaining bespoke runners (`olmocr_bench_reducto`, BFCL) keep their own `--limit`/`--evaluate-only` flags. Two extra setup steps for specific benchmarks: From 4c15a35ce2de8bafe37b923186a95c299b09e1eb Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 17:03:43 +0530 Subject: [PATCH 23/38] =?UTF-8?q?chore:=20stop=20tracking=20results/=20?= =?UTF-8?q?=E2=80=94=20run=20artifacts,=20never=20in=20git?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit results/ (benchmark outputs, incl. Spider2/OCRBench responses, ~200MB) is now gitignored entirely and purged from this branch's history (git filter-branch). Files stay on disk locally; regenerate via a run or bench migrate-results. Note: the report + CI baseline previously read committed metrics.json; with results/ untracked they read local files (report) / show 'new' vs no baseline (ci_compare) — no committed benchmark data. --- .gitignore | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index f4f3da8..c125a11 100644 --- a/.gitignore +++ b/.gitignore @@ -216,10 +216,10 @@ external/ benchmarks/spider2_lite/data/ benchmarks/olmocr/bench/full_data/ -# New-contract per-run responses (large; regenerable via `bench migrate-results` -# from the committed legacy flat results, or re-produced by a fresh run). The -# canonical metrics.json under results/// IS committed. -results/*/*/responses.jsonl +# Benchmark outputs are never tracked — they are run artifacts, not source, and +# large (hundreds of MB incl. Spider2/OCRBench responses). Kept on disk locally; +# regenerate with a run or `bench migrate-results`. +results/ # OCRBench spotting-eval scratch (generated during scoring) benchmarks/ocrbench_v2/eval_scripts/spotting_eval/gt/ From e416ef1d9f62096e28372a598b848511f8150d5f Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 17:11:17 +0530 Subject: [PATCH 24/38] docs: trim README to a scannable overview Collapse the 10 per-benchmark sections (each repeating the CLI 3-4x) into one benchmarks table + a single run example. Deep gotchas (per-benchmark one-time setup, Fireworks reasoning caveat, BFCL sharp edges) moved into
so they're available but not overwhelming. 255 -> 101 lines. --- README.md | 319 +++++++++++------------------------------------------- 1 file changed, 62 insertions(+), 257 deletions(-) diff --git a/README.md b/README.md index 6387d33..5f6cb97 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ # Interfaze complete benchmark scripts -[Full break down blog](https://interfaze.ai/blog/interfaze-a-new-model-architecture-built-for-high-accuracy-at-scale) | [Leaderboard](https://interfaze.ai/leaderboards) +[Blog](https://interfaze.ai/blog/interfaze-a-new-model-architecture-built-for-high-accuracy-at-scale) · [Leaderboard](https://interfaze.ai/leaderboards) -Runner scripts for the public benchmarks Interfaze is evaluated on. Each -benchmark lives in its own directory under `benchmarks/`. +Runner scripts for the public benchmarks Interfaze is evaluated on. ## Setup @@ -11,286 +10,92 @@ benchmark lives in its own directory under `benchmarks/`. uv sync ``` -Create a `.env` in the repo root with whichever provider keys you plan to use: +Add a `.env` with the provider keys you'll use: -``` -INTERFAZE_API_KEY=... -OPENAI_API_KEY=... -ANTHROPIC_API_KEY=... -GEMINI_KEY=... -OPENROUTER_API_KEY=... -FIREWORKS_API_KEY=... -``` - -Benchmarks run through one CLI: `uv run python -m bench_core run --target --benchmark ` (see `python -m bench_core list-targets`). `--sample N` is a smoke test; re-running resumes from the per-run checkpoint (completed samples are skipped). The remaining bespoke runners (`olmocr_bench_reducto`, BFCL) keep their own `--limit`/`--evaluate-only` flags. - -Two extra setup steps for specific benchmarks: - -```bash -# olmOCR-bench scoring renders equations in headless chromium -uv run python -m playwright install chromium - -# OCRBench v2 scoring needs NLTK corpora. Run it from OUTSIDE the repo root: -# nltk blocks imports resolving inside the CWD, and .venv lives in the project, -# so `import nltk` from here trips its guard. -(cd /tmp && uv run --project "$OLDPWD" python -c " -import nltk; [nltk.download(p, quiet=True) for p in ('wordnet','omw-1.4','punkt','punkt_tab')]") - -# GPQA's dataset (Idavidrein/gpqa) is gated — accept the terms on the Hub, then -hf auth login -``` - -## BFCL (Berkeley Function Calling Leaderboard) - -Links: [repo](https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard) · [leaderboard](https://gorilla.cs.berkeley.edu/leaderboard.html) - -BFCL lives in its own third-party checkout under `external/` (gitignored) with a -separate venv — its pins conflict with this repo's. `scripts/setup_bfcl.sh` is -the durable artifact: it clones, builds the venv, writes BFCL's `.env`, and -registers the Fireworks models idempotently. Re-run it any time. - -```bash -scripts/setup_bfcl.sh # once; safe to re-run - -# OpenRouter-hosted models (registered by the setup script) -scripts/bfcl.sh generate --model openrouter-qwen3.5-35b-a3b-FC \ - --test-category simple_python --num-threads 4 -scripts/bfcl.sh evaluate --model openrouter-qwen3.5-35b-a3b-FC --test-category simple_python - -scripts/bfcl.sh generate --model openrouter-gemma-4-31b-it-FC \ - --test-category simple_python --num-threads 4 -scripts/bfcl.sh evaluate --model openrouter-gemma-4-31b-it-FC --test-category simple_python -``` - -`evaluate` re-scores from saved responses, so a registration fix (see -`underscore_to_dot` below) costs nothing to apply — no regeneration. - -Use `scripts/bfcl.sh` rather than the `bfcl` CLI directly — it exports -`BFCL_PROJECT_ROOT` (which BFCL needs *before* start, since it derives its -`.env` path from it) and normalises the lowercase `fireworks_api_key` in this -repo's `.env` to the uppercase name BFCL's handler reads. - -Three sharp edges worth knowing: - -- This is BFCL **v4**: the category is `simple_python`, not `simple`. -- A Fireworks run needs a non-empty `OPENAI_API_KEY` even though it never calls - OpenAI — `FireworksHandler.__init__` calls `super().__init__()`, and the parent - `OpenAICompletionsHandler` builds an OpenAI client that raises without one. - The setup script writes a placeholder. -- `soundfile` is an undeclared transitive dependency (`model_config.py` imports - every handler eagerly → `qwen.py` → `qwen_agent` → `soundfile`). Without it, - loading the model registry fails, so *no* model can run. -- **`underscore_to_dot` must match the provider, or correct answers score wrong.** - BFCL's test set uses dotted function names (`math.factorial`), but OpenAI-style - tool schemas forbid `.`, so OpenRouter sanitises them and the model replies - `math_factorial`. With the flag set wrong, every such call fails as - `wrong_func_name`: measured 55.0% vs 93.8% for the same responses. OpenRouter - needs `True`; Fireworks passes dots through and needs `False`. The setup script - encodes this per provider and *refreshes* existing entries on re-run, so a - corrected flag actually takes effect. - -Measured on `simple_python` (400 cases, function-calling mode): - -| Model | Accuracy | -|---|---| -| Qwen3.5-35B-A3B (OpenRouter) | **93.75%** (375/400) | -| Qwen3.7-Plus (Fireworks) | **92.75%** (371/400) | -| Gemma-4-31B-IT (OpenRouter) | **92.25%** (369/400) | - -### Fireworks / Inkling - -`--provider fireworks` (or the `*_fireworks` runners) targets Fireworks-hosted -models over their OpenAI-compatible endpoint. Inkling takes text, image, and -audio input, so it runs on every benchmark here: - -```bash -uv run python -m bench_core run --target inkling --benchmark ocrbench_v2 -uv run python -m bench_core run --target inkling --benchmark olmocr -uv run python -m bench_core run --target inkling --benchmark gpqa -uv run python -m bench_core run --target inkling --benchmark mmmlu -uv run python -m bench_core run --target inkling --benchmark mmmu_pro --variant standard -uv run python -m bench_core run --target inkling --benchmark mmmu_pro --variant vision -uv run python -m bench_core run --target inkling --benchmark refcoco --variant val -uv run python -m bench_core run --target inkling --benchmark asr -uv run python -m bench_core run --target inkling --benchmark spider2 -``` - -Two caveats when reading Fireworks numbers: - -- **Reasoning can't be turned off.** Inkling's `reasoning_effort` accepts - `none|low|medium|high|xhigh|max`; `none` is a floor, not a disable, and still - emits reasoning tokens (measured: ~240 on MMMU-Pro standard, ~1100 on - MMMU-Pro vision). Runs tagged `reasoningoff` here are therefore *not* - like-for-like with providers whose thinking is genuinely disabled. Per-sample - `reasoning_tokens` are recorded in the output JSONL so the real spend is - auditable. -- **`inkling-small` is not on serverless.** Only - `accounts/fireworks/models/inkling` is serverless; `inkling-small` needs an - on-demand dedicated deployment first. Once deployed, pass it via `--model` - and every runner writes to its own separate checkpoint. - ---- - -## OCRBench v2 - -Links: [paper](https://arxiv.org/abs/2501.00321) · [repo](https://github.com/Yuliang-Liu/MultimodalOCR/tree/main/OCRBench_v2) - -```bash -# Any target (10k tasks, macro-of-category en_overall) -uv run python -m bench_core run --target gpt-5.5 --benchmark ocrbench_v2 -uv run python -m bench_core run --target gemini-3.7-flash --benchmark ocrbench_v2 -uv run python -m bench_core run --target inkling --benchmark ocrbench_v2 -``` - ---- - -## olmOCR-Bench - -Links: [repo](https://github.com/allenai/olmocr/tree/main/olmocr/bench) · [dataset](https://huggingface.co/datasets/allenai/olmOCR-bench) - -```bash -# Any target (renders each PDF page, scores via the external olmocr.bench scorer; -# results land in logs/olmocr_.log, which the report reads) -uv run python -m bench_core run --target gemini-3.7-flash --benchmark olmocr -uv run python -m bench_core run --target inkling --benchmark olmocr - -# Reducto (a bespoke document-parsing API, not a chat model) keeps its own runner -uv run -m benchmarks.olmocr.olmocr_bench_reducto +```env +INTERFAZE_API_KEY=… OPENAI_API_KEY=… ANTHROPIC_API_KEY=… +GEMINI_KEY=… OPENROUTER_API_KEY=… FIREWORKS_API_KEY=… ``` ---- +## Run a benchmark -## RefCOCO (Object Detection) - -Links: [RefCOCO/RefCOCO+ paper](https://arxiv.org/abs/1608.00272) · [RefCOCOg paper](https://arxiv.org/abs/1511.02283) · [dataset](https://huggingface.co/datasets/lmms-lab/RefCOCO) - -Metric: Acc@IoU=0.5 on the referring-expression bounding box. +Everything runs through one CLI (`bench_core`). Models are defined in +`bench_core/targets.yaml`; add one there to benchmark a new model — no code. ```bash -# --variant selects dataset + split: base RefCOCO (val|testA|testB|test), or -# RefCOCO+ (plus-val|plus-testA|plus-testB) / RefCOCOg (g-val|g-test). Metrics -# include a strict Acc@0.5 plus a format-tolerant `oracle` upper bound. -uv run python -m bench_core run --target gpt-5.5 --benchmark refcoco --variant val -uv run python -m bench_core run --target gemini-3.7-flash --benchmark refcoco --variant testA -uv run python -m bench_core run --target inkling --benchmark refcoco --variant plus-testB -uv run python -m bench_core run --target inkling --benchmark refcoco --variant g-val +uv run python -m bench_core list-targets # what's available +uv run python -m bench_core run --target inkling --benchmark gpqa # a full run +uv run python -m bench_core run --target gpt-5.5 --benchmark ocrbench_v2 --sample 20 # smoke +uv run python scripts/report_scores.py # view scores ``` ---- - -## VoxPopuli-Cleaned-AA (ASR) +`--sample N` runs the first N samples (smoke); omit it for the full benchmark. +Re-running resumes from the checkpoint. Add `--reasoning off|low|medium|high` to +override the target's default. -Links: [dataset](https://huggingface.co/datasets/ArtificialAnalysis/VoxPopuli-Cleaned-AA) +| Benchmark | `--benchmark` | Notes | Links | +|---|---|---|---| +| GPQA Diamond | `gpqa` | dataset is gated → `hf auth login` first | [paper](https://arxiv.org/abs/2311.12022) · [data](https://huggingface.co/datasets/Idavidrein/gpqa) | +| MMMLU | `mmmlu` | `--variant lite` (default) or `full` | [data](https://huggingface.co/datasets/openai/MMMLU) | +| MMMU-Pro | `mmmu_pro` | `--variant standard` or `vision` | [paper](https://arxiv.org/abs/2409.02813) · [data](https://huggingface.co/datasets/MMMU/MMMU_Pro) | +| OCRBench v2 | `ocrbench_v2` | 10k tasks; needs NLTK corpora † | [paper](https://arxiv.org/abs/2501.00321) · [repo](https://github.com/Yuliang-Liu/MultimodalOCR/tree/main/OCRBench_v2) | +| olmOCR | `olmocr` | needs poppler + chromium † | [repo](https://github.com/allenai/olmocr/tree/main/olmocr/bench) · [data](https://huggingface.co/datasets/allenai/olmOCR-bench) | +| RefCOCO | `refcoco` | `--variant val\|testA\|testB\|test\|plus-*\|g-*` | [paper](https://arxiv.org/abs/1608.00272) · [data](https://huggingface.co/datasets/lmms-lab/RefCOCO) | +| ASR (VoxPopuli) | `asr` | WER, any audio-capable target | [data](https://huggingface.co/datasets/ArtificialAnalysis/VoxPopuli-Cleaned-AA) | +| Spider2-Lite | `spider2` | run `fetch_data` first (~4GB) † | [repo](https://github.com/xlang-ai/Spider2) · [paper](https://arxiv.org/abs/2411.07763) | -Metric: WER with Whisper-style text normalization. +
+† one-time setup for some benchmarks ```bash -# Any audio-capable target (the right audio wire-shape is chosen per provider) -uv run python -m bench_core run --target interfaze-beta --benchmark asr -uv run python -m bench_core run --target gemini-3.7-flash --benchmark asr -uv run python -m bench_core run --target inkling --benchmark asr -``` - ---- - -## MMMLU (Multilingual MMLU) - -Links: [dataset](https://huggingface.co/datasets/openai/MMMLU) +# olmOCR — headless chromium for equation rendering +uv run python -m playwright install chromium -14 languages, exact-match accuracy macro-averaged across languages. +# OCRBench v2 — NLTK corpora. Run from OUTSIDE the repo (nltk trips on a CWD .venv): +(cd /tmp && uv run --project "$OLDPWD" python -c \ + "import nltk; [nltk.download(p, quiet=True) for p in ('wordnet','omw-1.4','punkt','punkt_tab')]") -```bash -# 14 languages, macro-averaged. --variant lite (opencompass/mmmlu_lite, ~20k, -# DEFAULT, matches the archived leaderboard numbers) or full (openai/MMMLU, ~196k). -uv run python -m bench_core run --target gpt-5.5 --benchmark mmmlu # lite -uv run python -m bench_core run --target gemini-3.7-flash --benchmark mmmlu --reasoning high -uv run python -m bench_core run --target inkling --benchmark mmmlu --variant full # full MMLU +# Spider2 — clone + download the SQLite databases +uv run -m benchmarks.spider2_lite.fetch_data ``` +
---- +
+Reading Fireworks / Inkling numbers -## MMMU-Pro +Inkling's reasoning **can't be fully disabled** — `reasoning_effort` accepts +`none|low|…|max`, but `none` is a floor that still emits reasoning tokens, so +runs tagged `off` aren't like-for-like with providers that truly disable +thinking. Per-sample `reasoning_tokens` are recorded so the real spend is +auditable. `inkling-small` isn't on serverless (needs a dedicated deployment). +
-Links: [paper](https://arxiv.org/abs/2409.02813) · [dataset](https://huggingface.co/datasets/MMMU/MMMU_Pro) - -Two settings: `standard` (text + inline images) and `vision` (rendered question image). - -```bash -# Any target, standard or vision -uv run python -m bench_core run --target gemini-3.7-flash --benchmark mmmu_pro --variant standard -uv run python -m bench_core run --target gemini-3.7-flash --benchmark mmmu_pro --variant vision -uv run python -m bench_core run --target inkling --benchmark mmmu_pro --variant standard - -# Run on Modal instead of locally -bash benchmarks/mmmu_pro/run_full.sh -bash benchmarks/mmmu_pro/run_smoke.sh -``` - ---- - -## GPQA Diamond +## BFCL (Berkeley Function Calling Leaderboard) -Links: [paper](https://arxiv.org/abs/2311.12022) · [dataset](https://huggingface.co/datasets/Idavidrein/gpqa) (config: `gpqa_diamond`) +Separate third-party harness in its own venv ([repo](https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard) · [leaderboard](https://gorilla.cs.berkeley.edu/leaderboard.html)). ```bash -# GPQA runs through the unified CLI. Provider, model, and reasoning defaults come -# from the target (bench_core/targets.yaml); pick any target by name. -uv run python -m bench_core list-targets -uv run python -m bench_core run --target gpt-5.5 --benchmark gpqa -uv run python -m bench_core run --target gemini-3.7-flash --benchmark gpqa --reasoning high -uv run python -m bench_core run --target grok-4.3 --benchmark gpqa -uv run python -m bench_core run --target inkling --benchmark gpqa +scripts/setup_bfcl.sh # once; safe to re-run +scripts/bfcl.sh generate --model openrouter-qwen3.5-35b-a3b-FC --test-category simple_python --num-threads 4 +scripts/bfcl.sh evaluate --model openrouter-qwen3.5-35b-a3b-FC --test-category simple_python ``` ---- +
+BFCL gotchas -## Spider 2.0-Lite (SQLite subset, N=135) +- Use `scripts/bfcl.sh`, not `bfcl` directly (it sets `BFCL_PROJECT_ROOT` and the uppercase `FIREWORKS_API_KEY`). +- BFCL **v4**: category is `simple_python`, not `simple`. +- Fireworks runs need a non-empty `OPENAI_API_KEY` placeholder (parent handler builds an OpenAI client); `soundfile` is an undeclared dep the setup script installs. +- **`underscore_to_dot` must match the provider** or correct calls score as `wrong_func_name` (measured 55% vs 94%). OpenRouter → `True`, Fireworks → `False`; the setup script sets it per provider. -Links: [repo](https://github.com/xlang-ai/Spider2) · [paper](https://arxiv.org/abs/2411.07763) - -Text-to-SQL with execution-accuracy scoring against per-example SQLite databases. - -```bash -# One-time setup: clone Spider2 + download SQLite databases -uv run -m benchmarks.spider2_lite.fetch_data # downloads the ~4GB data/ (required) - -# Run any target (execution accuracy over the 135 local SQLite instances) -uv run python -m bench_core run --target gpt-5.5 --benchmark spider2 -uv run python -m bench_core run --target inkling --benchmark spider2 -``` +
## Automation (GitHub Actions) -Three workflows under `.github/workflows/`: - -- **`tests.yml`** — the free gate on every push/PR: the offline `bench_core` suite - + lint. `test_targets.py` validates every `bench_core/targets.yaml` entry, so a - new model is checked here before it ever runs. -- **`benchmark.yml`** — runs benchmarks through `python -m bench_core run`. - - *Manual* (`workflow_dispatch`): pick `target`, `benchmarks` (csv or `all`), - `sample_size` (blank = full run), and an optional `reasoning` override. - - *Nightly* (`schedule`): smoke-runs every target flagged `ci_regression: true` - to catch a provider changing its API (an adapter breaking shows up as a crash, - not a silent drift). - - *On merge* (`push` to `main` touching `targets.yaml`): diffs the merge and runs - a **full** benchmark for the target(s) that changed. - - A `report` job diffs each run against the committed baseline - (`scripts/ci_compare.py`) and writes the score tables to the job summary. -- **`add-target.yml`** — the interactive "add a model" form (Actions → add-target → - Run workflow): enter `name` / `provider` / `model_id` / optional - `capabilities_json` / `ci_regression`. It validates the entry, appends it to - `targets.yaml`, and **opens a PR** — it does not run anything itself. - -**Two ways to add a model, both gated on merge:** -1. *Guided:* run `add-target` from the UI → it opens a PR. -2. *By hand:* edit `bench_core/targets.yaml` in a PR. - -Either way, `tests.yml` validates the entry on the PR, and **merging the PR** is -what triggers `benchmark.yml` to run the new target (full run). No code, and the -benchmark only runs once a human has merged. - -**Secrets required:** `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GEMINI_API_KEY`, -`FIREWORKS_API_KEY`, `OPENROUTER_API_KEY`, `INTERFAZE_API_KEY`, `HF_TOKEN`. +- **`tests.yml`** — offline suite + lint on every push/PR (validates new `targets.yaml` entries). +- **`benchmark.yml`** — runs a benchmark: manual dispatch (pick target/benchmarks), weekly smoke of `ci_regression` targets, and a **full run on merge** of a `targets.yaml` change. +- **`add-target.yml`** — a UI form to add a model that opens a PR (doesn't run anything). -**Not in CI:** `olmocr` (needs poppler + playwright + the full dataset to score) -and `spider2` (needs the ~4GB `data/`); run those locally or on a self-hosted runner. +Add a model via the form or by editing `targets.yaml` in a PR; the benchmark +runs **only once the PR is merged**. Needs the provider secrets + `HF_TOKEN` set +in the repo. `olmocr` and `spider2` aren't in CI (heavy deps / ~4GB data). From c2e22e33fa9ff0a88a62a141da2fa14f0db7ceb7 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 17:14:28 +0530 Subject: [PATCH 25/38] chore: remove BFCL from the repo Deletes scripts/setup_bfcl.sh, scripts/bfcl.sh, the README BFCL section, and the now-unused external/ gitignore entry (external/ was BFCL's checkout + venv; removed from disk, ~1.2GB). The repo now covers the 8 bench_core benchmarks. 174 tests pass. --- .gitignore | 3 - README.md | 30 ----- scripts/bfcl.sh | 47 -------- scripts/setup_bfcl.sh | 251 ------------------------------------------ 4 files changed, 331 deletions(-) delete mode 100755 scripts/bfcl.sh delete mode 100755 scripts/setup_bfcl.sh diff --git a/.gitignore b/.gitignore index c125a11..bfbac09 100644 --- a/.gitignore +++ b/.gitignore @@ -209,9 +209,6 @@ __marimo__/ # olmOCR-Bench dataset + per-candidate prediction outputs (large, downloaded from HF) benchmarks/olmocr_bench/olmocr_bench/ -# Third-party benchmark checkouts + their venvs (see scripts/setup_bfcl.sh) -external/ - # Downloaded benchmark datasets (large; fetched at runtime, not source) benchmarks/spider2_lite/data/ benchmarks/olmocr/bench/full_data/ diff --git a/README.md b/README.md index 5f6cb97..335c6eb 100644 --- a/README.md +++ b/README.md @@ -60,36 +60,6 @@ uv run -m benchmarks.spider2_lite.fetch_data ```
-
-Reading Fireworks / Inkling numbers - -Inkling's reasoning **can't be fully disabled** — `reasoning_effort` accepts -`none|low|…|max`, but `none` is a floor that still emits reasoning tokens, so -runs tagged `off` aren't like-for-like with providers that truly disable -thinking. Per-sample `reasoning_tokens` are recorded so the real spend is -auditable. `inkling-small` isn't on serverless (needs a dedicated deployment). -
- -## BFCL (Berkeley Function Calling Leaderboard) - -Separate third-party harness in its own venv ([repo](https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard) · [leaderboard](https://gorilla.cs.berkeley.edu/leaderboard.html)). - -```bash -scripts/setup_bfcl.sh # once; safe to re-run -scripts/bfcl.sh generate --model openrouter-qwen3.5-35b-a3b-FC --test-category simple_python --num-threads 4 -scripts/bfcl.sh evaluate --model openrouter-qwen3.5-35b-a3b-FC --test-category simple_python -``` - -
-BFCL gotchas - -- Use `scripts/bfcl.sh`, not `bfcl` directly (it sets `BFCL_PROJECT_ROOT` and the uppercase `FIREWORKS_API_KEY`). -- BFCL **v4**: category is `simple_python`, not `simple`. -- Fireworks runs need a non-empty `OPENAI_API_KEY` placeholder (parent handler builds an OpenAI client); `soundfile` is an undeclared dep the setup script installs. -- **`underscore_to_dot` must match the provider** or correct calls score as `wrong_func_name` (measured 55% vs 94%). OpenRouter → `True`, Fireworks → `False`; the setup script sets it per provider. - -
- ## Automation (GitHub Actions) - **`tests.yml`** — offline suite + lint on every push/PR (validates new `targets.yaml` entries). diff --git a/scripts/bfcl.sh b/scripts/bfcl.sh deleted file mode 100755 index 5a74cc2..0000000 --- a/scripts/bfcl.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# Thin wrapper around BFCL's CLI that fixes up the environment first. -# -# scripts/bfcl.sh generate --model fireworks-qwen3p7-plus-FC --test-category simple_python -# scripts/bfcl.sh evaluate --model fireworks-qwen3p7-plus-FC --test-category simple_python -# -# Two reasons this exists rather than calling `bfcl` directly: -# -# 1. BFCL derives DOTENV_PATH from BFCL_PROJECT_ROOT, which it reads from the -# *process* environment — so BFCL_PROJECT_ROOT cannot be supplied by the -# .env it is used to locate. It has to be exported before the CLI starts. -# 2. This project's .env spells the Fireworks key lowercase, while BFCL's -# handler does a bare os.getenv("FIREWORKS_API_KEY"). We normalise here. -# -# Run scripts/setup_bfcl.sh once first. - -set -euo pipefail -cd "$(dirname "$0")/.." -ROOT="$PWD" - -VENV="$ROOT/external/bfcl-venv" -RUNS="$ROOT/external/bfcl-runs" - -if [ ! -x "$VENV/bin/bfcl" ]; then - echo "bfcl not installed — run scripts/setup_bfcl.sh first" >&2 - exit 1 -fi - -export BFCL_PROJECT_ROOT="$RUNS" -export FIREWORKS_API_KEY="${FIREWORKS_API_KEY:-$( - grep -iE '^[[:space:]]*fireworks_api_key[[:space:]]*=' "$ROOT/.env" 2>/dev/null \ - | head -1 | cut -d= -f2- | tr -d ' "'"'" || true -)}" -export OPENROUTER_API_KEY="${OPENROUTER_API_KEY:-$( - grep -iE '^[[:space:]]*openrouter_api_key[[:space:]]*=' "$ROOT/.env" 2>/dev/null \ - | head -1 | cut -d= -f2- | tr -d ' "'"'" || true -)}" -# See setup_bfcl.sh — needed only so the parent handler's client can be built. -export OPENAI_API_KEY="${OPENAI_API_KEY:-placeholder-unused-by-fireworks-handler}" - -if [ -z "$FIREWORKS_API_KEY" ] && [ -z "$OPENROUTER_API_KEY" ]; then - echo "Neither FIREWORKS_API_KEY nor OPENROUTER_API_KEY found in $ROOT/.env" >&2 - exit 1 -fi - -cd "$RUNS" -exec "$VENV/bin/bfcl" "$@" diff --git a/scripts/setup_bfcl.sh b/scripts/setup_bfcl.sh deleted file mode 100755 index bd2fa91..0000000 --- a/scripts/setup_bfcl.sh +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env bash -# Set up the Berkeley Function Calling Leaderboard (BFCL) for Fireworks-hosted -# models, idempotently. -# -# scripts/setup_bfcl.sh -# -# Why a script and not a one-off: BFCL lives in a third-party clone under -# external/ (gitignored), so any edit made directly in that tree is lost on -# re-clone or a fresh checkout. This script IS the durable artifact — re-running -# it reproduces the whole setup, including the model registrations. -# -# It does four things: -# 1. clones gorilla (shallow) if absent -# 2. builds a SEPARATE venv — BFCL's pins conflict with this repo's -# (olmocr==0.4.27 and friends), so it must not share .venv -# 3. writes BFCL's own .env with FIREWORKS_API_KEY *uppercase*: BFCL's -# FireworksHandler does a bare os.getenv("FIREWORKS_API_KEY") with no -# fallback, and this project's .env spells the key lowercase -# 4. registers the Fireworks models in model_config.py, append-if-absent -# -# After it runs: -# external/bfcl-venv/bin/bfcl generate --model \ -# --test-category simple --num-threads 4 -# external/bfcl-venv/bin/bfcl evaluate --model --test-category simple - -set -euo pipefail -cd "$(dirname "$0")/.." -ROOT="$PWD" - -GORILLA_DIR="$ROOT/external/gorilla" -BFCL_DIR="$GORILLA_DIR/berkeley-function-call-leaderboard" -VENV="$ROOT/external/bfcl-venv" -RUNS="$ROOT/external/bfcl-runs" -CONFIG="$BFCL_DIR/bfcl_eval/constants/model_config.py" - -# ---------------------------------------------------------------- 1. clone -if [ ! -d "$BFCL_DIR" ]; then - echo "[clone] ShishirPatil/gorilla (depth 1)" - mkdir -p "$ROOT/external" - git clone --depth 1 https://github.com/ShishirPatil/gorilla.git "$GORILLA_DIR" -else - echo "[skip] gorilla already cloned at $GORILLA_DIR" -fi - -# ---------------------------------------------------------------- 2. venv -if [ ! -x "$VENV/bin/bfcl" ]; then - echo "[venv] creating $VENV and installing bfcl_eval (editable)" - uv venv "$VENV" --python 3.12 >/dev/null - VIRTUAL_ENV="$VENV" uv pip install --python "$VENV/bin/python" -e "$BFCL_DIR" >/dev/null - # soundfile is an undeclared transitive dep: model_config.py imports every - # handler eagerly, including qwen.py -> qwen_agent -> soundfile. Without it, - # merely LOADING the model registry raises ModuleNotFoundError, so no model - # can run — not just the Qwen-agent ones. - VIRTUAL_ENV="$VENV" uv pip install --python "$VENV/bin/python" soundfile >/dev/null -else - echo "[skip] bfcl already installed in $VENV" -fi - -# ---------------------------------------------------------------- 3. env -mkdir -p "$RUNS" -# Accept either casing from the project .env, write the uppercase name BFCL wants. -FW_KEY="$( - grep -iE '^[[:space:]]*fireworks_api_key[[:space:]]*=' "$ROOT/.env" 2>/dev/null \ - | head -1 | cut -d= -f2- | tr -d ' "'"'" || true -)" -if [ -z "${FW_KEY:-}" ]; then - echo "[warn] no fireworks_api_key found in $ROOT/.env — BFCL will 401 until you add it" -fi -OR_KEY="$( - grep -iE '^[[:space:]]*openrouter_api_key[[:space:]]*=' "$ROOT/.env" 2>/dev/null \ - | head -1 | cut -d= -f2- | tr -d ' "'"'" || true -)" -BFCL_ENV="$RUNS/.env" -{ - echo "# generated by scripts/setup_bfcl.sh — do not edit by hand" - echo "FIREWORKS_API_KEY=${FW_KEY:-}" - echo "BFCL_PROJECT_ROOT=$RUNS" - # FireworksHandler.__init__ calls super().__init__() first, and - # OpenAICompletionsHandler builds an OpenAI() client that raises without - # OPENAI_API_KEY — before the Fireworks client replaces it. So a Fireworks run - # cannot start without *some* value here. It is never used for inference. - echo "OPENAI_API_KEY=placeholder-unused-by-fireworks-handler" - echo "OPENROUTER_API_KEY=${OR_KEY:-}" -} > "$BFCL_ENV" -cp "$BFCL_ENV" "$BFCL_DIR/.env" -echo "[env] wrote $BFCL_ENV (and a copy at $BFCL_DIR/.env)" - -# ------------------------------------------------- 4a. OpenRouter handler -# BFCL ships no OpenRouter handler. OpenRouter is OpenAI-compatible, so this is -# a minimal subclass that swaps in the client — it deliberately keeps the -# parent's OPENAI_COMPLETIONS model_style, which already emits the standard -# {"type": "function", ...} tool shape OpenRouter expects. -OR_HANDLER="$BFCL_DIR/bfcl_eval/model_handler/api_inference/openrouter.py" -if [ ! -f "$OR_HANDLER" ]; then - cat > "$OR_HANDLER" <<'PY' -import os - -from openai import OpenAI - -from bfcl_eval.model_handler.api_inference.openai_completion import ( - OpenAICompletionsHandler, -) - - -class OpenRouterHandler(OpenAICompletionsHandler): - """Any OpenRouter-hosted model over its OpenAI-compatible endpoint. - - Added by this repo's scripts/setup_bfcl.sh — not part of upstream BFCL. - Note the parent __init__ still builds an OpenAI() client first and raises - without OPENAI_API_KEY, hence the placeholder written into BFCL's .env. - """ - - def __init__(self, model_name, temperature, registry_name, is_fc_model, **kwargs) -> None: - super().__init__(model_name, temperature, registry_name, is_fc_model, **kwargs) - self.client = OpenAI( - base_url="https://openrouter.ai/api/v1", - api_key=os.getenv("OPENROUTER_API_KEY"), - ) -PY - echo "[handler] wrote $OR_HANDLER" -else - echo "[skip] OpenRouter handler already present" -fi - -# Make sure model_config.py imports it (idempotent). -python3 - "$CONFIG" <<'PY' -import sys -from pathlib import Path -p = Path(sys.argv[1]); src = p.read_text() -imp = ("from bfcl_eval.model_handler.api_inference.openrouter import (\n" - " OpenRouterHandler,\n)\n") -if "OpenRouterHandler" not in src: - anchor = "from bfcl_eval.model_handler.api_inference.fireworks import FireworksHandler\n" - assert anchor in src, "fireworks import anchor not found" - src = src.replace(anchor, anchor + imp, 1) - p.write_text(src) - print("[register] added OpenRouterHandler import") -else: - print("[skip] OpenRouterHandler already imported") -PY - -# ---------------------------------------------------------------- 4. register -# One entry per model, appended into api_inference_model_map only if its registry -# name is not already present. -# -# is_fc_model: NOT verifiable while both models 404. Registered in prompt mode -# (is_fc_model=False) as the safer default — an FC registration against a model -# that silently ignores `tools=` yields a near-zero score that reads like a -# capability result. Re-confirm on the first real run and flip to the -FC entry -# if the model does emit native tool calls. -python3 - "$CONFIG" <<'PY' -import sys, re -from pathlib import Path - -path = Path(sys.argv[1]) -src = path.read_text() - -ENTRIES = { - # registry name -> (model id, display name, org, license, is_fc, underscore_to_dot) - # - # underscore_to_dot: True when the model returns sanitised function names - # (math_factorial) instead of the dotted originals (math.factorial). This is - # provider-specific, not cosmetic: with the wrong value the checker rejects - # CORRECT calls as wrong_func_name. Verified empirically per provider — - # OpenRouter sanitises (OpenAI tool schemas forbid "." in names), Fireworks - # passes dots through. - "fireworks-qwen3p5-35b-a3b": ( - "accounts/fireworks/models/qwen3p5-35b-a3b", - "Qwen3.5-35B-A3B (Fireworks, prompt)", "Qwen", "apache-2.0", False, False, - ), - "fireworks-qwen3p5-35b-a3b-FC": ( - "accounts/fireworks/models/qwen3p5-35b-a3b", - "Qwen3.5-35B-A3B (Fireworks, FC)", "Qwen", "apache-2.0", True, False, - ), - "fireworks-gemma-4-31b-it": ( - "accounts/fireworks/models/gemma-4-31b-it", - "Gemma-4-31B-IT (Fireworks, prompt)", "Google", "gemma-terms-of-use", False, False, - ), - "fireworks-gemma-4-31b-it-FC": ( - "accounts/fireworks/models/gemma-4-31b-it", - "Gemma-4-31B-IT (Fireworks, FC)", "Google", "gemma-terms-of-use", True, False, - ), - # --- OpenRouter: both models ARE available here, unlike Fireworks serverless. - # is_fc_model=True is probe-verified: each emits native tool_calls when given - # `tools`, and OpenRouter advertises tools in supported_parameters for both. - "openrouter-qwen3.5-35b-a3b-FC": ( - "qwen/qwen3.5-35b-a3b", - "Qwen3.5-35B-A3B (OpenRouter, FC)", "Qwen", "apache-2.0", True, True, - ), - "openrouter-gemma-4-31b-it-FC": ( - "google/gemma-4-31b-it", - "Gemma-4-31B-IT (OpenRouter, FC)", "Google", "gemma-terms-of-use", True, True, - ), - # An available serverless model, used only to validate the plumbing. - "fireworks-qwen3p7-plus-FC": ( - "accounts/fireworks/models/qwen3p7-plus", - "Qwen3.7-Plus (Fireworks, FC)", "Qwen", "proprietary", True, False, - ), -} - -added, updated = [], [] -for name, (model_id, display, org, lic, is_fc, us2dot) in ENTRIES.items(): - # Remove any existing block for this name so a corrected field (e.g. - # underscore_to_dot) actually takes effect on re-run. Append-only would - # silently keep the stale value. - existing = re.search( - r'^ "' + re.escape(name) + r'": ModelConfig\(\n(?:.*?\n)*? \),\n', - src, re.M) - if existing: - src = src[:existing.start()] + src[existing.end():] - updated.append(name) - block = ( - f' "{name}": ModelConfig(\n' - f' model_name="{model_id}",\n' - f' display_name="{display}",\n' - f' url="{("https://openrouter.ai/" + model_id) if name.startswith("openrouter-") else ("https://fireworks.ai/models/fireworks/" + model_id.rsplit("/", 1)[-1])}",\n' - f' org="{org}",\n' - f' license="{lic}",\n' - f' model_handler={"OpenRouterHandler" if name.startswith("openrouter-") else "FireworksHandler"},\n' - f' input_price=None,\n' - f' output_price=None,\n' - f' is_fc_model={is_fc},\n' - f' underscore_to_dot={us2dot},\n' - f' ),\n' - ) - # Append just before the closing brace of api_inference_model_map. - m = re.search(r"^api_inference_model_map = \{$", src, re.M) - if not m: - sys.exit("could not find api_inference_model_map") - close = src.index("\n}\n", m.end()) - src = src[:close + 1] + block + src[close + 1:] - added.append(name) - -if added: - path.write_text(src) - fresh = [a for a in added if a not in updated] - if fresh: - print("[register] added: " + ", ".join(fresh)) - if updated: - print("[register] refreshed: " + ", ".join(updated)) -PY - -echo -echo "Setup complete. Registered models:" -"$VENV/bin/python" - <<'PY' -from bfcl_eval.constants.model_config import MODEL_CONFIG_MAPPING -for k in sorted(MODEL_CONFIG_MAPPING): - if k.startswith(("fireworks-", "openrouter-")): - c = MODEL_CONFIG_MAPPING[k] - print(f" {k:34} -> {c.model_name:48} is_fc={c.is_fc_model}") -PY From 821a99ea93e221e7292d01660471ff8770d54281 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 17:36:53 +0530 Subject: [PATCH 26/38] fix(cli): load .env so provider keys are picked up on live runs The old per-benchmark runners each called load_dotenv(); the new bench_core CLI did not, so 'python -m bench_core run' failed with 'no API key' despite keys in .env. main() now loads .env (python-dotenv, already a dep). Found via a live run. --- bench_core/cli.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bench_core/cli.py b/bench_core/cli.py index c82b806..99a9651 100644 --- a/bench_core/cli.py +++ b/bench_core/cli.py @@ -140,6 +140,14 @@ def cmd_migrate_results(args) -> None: def main(argv=None) -> None: + # pick up provider keys from a repo-root .env (adapters read os.getenv) + try: + from dotenv import load_dotenv + + load_dotenv() + except ImportError: + pass + p = argparse.ArgumentParser(prog="bench") sub = p.add_subparsers(dest="cmd", required=True) From ed2c47a4e6e9b63b45b0acc043d25c3f1b82d1f7 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Sun, 16 Aug 2026 17:38:31 +0530 Subject: [PATCH 27/38] docs: reflow README --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 335c6eb..65bec56 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,4 @@ uv run -m benchmarks.spider2_lite.fetch_data - **`benchmark.yml`** — runs a benchmark: manual dispatch (pick target/benchmarks), weekly smoke of `ci_regression` targets, and a **full run on merge** of a `targets.yaml` change. - **`add-target.yml`** — a UI form to add a model that opens a PR (doesn't run anything). -Add a model via the form or by editing `targets.yaml` in a PR; the benchmark -runs **only once the PR is merged**. Needs the provider secrets + `HF_TOKEN` set -in the repo. `olmocr` and `spider2` aren't in CI (heavy deps / ~4GB data). +Add a model via the form or by editing `targets.yaml` in a PR; the benchmark runs **only once the PR is merged**. Needs the provider secrets + `HF_TOKEN` set in the repo. `olmocr` and `spider2` aren't in CI (heavy deps / ~4GB data). From 8d55c09d5b5b75162d7701e3034abc89a4b0b5dd Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Mon, 17 Aug 2026 07:54:06 +0530 Subject: [PATCH 28/38] feat(bench): stream heavy image benchmarks on --sample; isolate smokes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --sample now streams only the first N rows (bench_core/datautil.load_rows) instead of downloading the whole split, so smoking a ~10k-image benchmark (ocrbench_v2) is cheap. Wired into mmmu_pro, refcoco, ocrbench_v2 — ocrbench keeps its lazy-by-index full path (materializing 10k decoded images would OOM); smokes embed the streamed image via a dual 'image'/'idx' path in build_request. Sampled runs write under results/_smoke/ so they can't clobber or shadow a full run's metrics (results/ is gitignored → a clobbered full score is unrecoverable) and stay out of report_scores (discover() globs depth-2 only). Verified live: mmmu_pro --sample 3 (gemini-2.5-flash) streamed 3 rows, real API calls (reasoning_tokens present), landed in results/_smoke/, invisible to discover(). Note: README also carries an unrelated title/.env-format tweak. --- README.md | 24 ++++++++++---- bench_core/cli.py | 6 +++- bench_core/datautil.py | 18 +++++++++++ benchmarks/mmmu_pro/bench.py | 5 ++- benchmarks/obj_detection/bench.py | 5 ++- benchmarks/ocrbench_v2/bench.py | 52 ++++++++++++++++++++----------- tests/test_datautil.py | 48 ++++++++++++++++++++++++++++ tests/test_results.py | 13 ++++++++ 8 files changed, 139 insertions(+), 32 deletions(-) create mode 100644 bench_core/datautil.py create mode 100644 tests/test_datautil.py diff --git a/README.md b/README.md index 65bec56..9d4302a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Interfaze complete benchmark scripts +# Interfaze complete benchmark [Blog](https://interfaze.ai/blog/interfaze-a-new-model-architecture-built-for-high-accuracy-at-scale) · [Leaderboard](https://interfaze.ai/leaderboards) @@ -13,8 +13,12 @@ uv sync Add a `.env` with the provider keys you'll use: ```env -INTERFAZE_API_KEY=… OPENAI_API_KEY=… ANTHROPIC_API_KEY=… -GEMINI_KEY=… OPENROUTER_API_KEY=… FIREWORKS_API_KEY=… +INTERFAZE_API_KEY=… +OPENAI_API_KEY=… +ANTHROPIC_API_KEY=… +GEMINI_KEY=… +OPENROUTER_API_KEY=… +FIREWORKS_API_KEY=… ``` ## Run a benchmark @@ -29,9 +33,16 @@ uv run python -m bench_core run --target gpt-5.5 --benchmark ocrbench_v2 --sampl uv run python scripts/report_scores.py # view scores ``` -`--sample N` runs the first N samples (smoke); omit it for the full benchmark. -Re-running resumes from the checkpoint. Add `--reasoning off|low|medium|high` to -override the target's default. +`--sample N` runs the first N samples — a *smoke*. It **streams** only those N +rows, so smoking a heavy image benchmark (e.g. `ocrbench_v2`, ~10k images) no +longer downloads the whole split first. Smokes write to `results/_smoke/…`, +isolated from full runs and hidden from `report_scores.py`, so they can't clobber +a real score. Omit `--sample` for the full benchmark. Re-running resumes from the +checkpoint. Add `--reasoning off|low|medium|high` to override the target's default. + +> A **full** image-benchmark run loads the whole split into RAM (RefCOCO `val` ≈ +> 8.8k images → several GB). Smoke with `--sample` first; if a full local run runs +> out of memory, see the lazy-by-index loader `ocrbench_v2` already uses. | Benchmark | `--benchmark` | Notes | Links | |---|---|---|---| @@ -58,6 +69,7 @@ uv run python -m playwright install chromium # Spider2 — clone + download the SQLite databases uv run -m benchmarks.spider2_lite.fetch_data ``` + ## Automation (GitHub Actions) diff --git a/bench_core/cli.py b/bench_core/cli.py index 99a9651..a7f3230 100644 --- a/bench_core/cli.py +++ b/bench_core/cli.py @@ -61,7 +61,11 @@ def cmd_run(args) -> None: result_name = bench.NAME print(f"Loading {result_name} samples...") - store = RunStore(result_name, target.name) + # sampled runs are smokes: isolate them under results/_smoke/ so they can + # never clobber or shadow a full run's metrics (results/ is gitignored, so a + # clobbered full-run metrics.json would be unrecoverable). + root = "results" if args.sample is None else "results/_smoke" + store = RunStore(result_name, target.name, root=root) d = bench.DEFAULTS result = asyncio.run( diff --git a/bench_core/datautil.py b/bench_core/datautil.py new file mode 100644 index 0000000..2102b4c --- /dev/null +++ b/bench_core/datautil.py @@ -0,0 +1,18 @@ +# Dataset loading that doesn't download a whole 10k-image split for a smoke. + +from __future__ import annotations + + +def load_rows( + dataset_id: str, + split: str, + sample_size: int | None = None, + config: str | None = None, +): + from datasets import load_dataset + + args = (dataset_id, config) if config else (dataset_id,) + if sample_size: + ds = load_dataset(*args, split=split, streaming=True) + return list(ds.take(sample_size)) + return list(load_dataset(*args, split=split)) diff --git a/benchmarks/mmmu_pro/bench.py b/benchmarks/mmmu_pro/bench.py index e483e17..8c92615 100644 --- a/benchmarks/mmmu_pro/bench.py +++ b/benchmarks/mmmu_pro/bench.py @@ -66,10 +66,9 @@ def _options_block(options: list[str]) -> str: def load_samples( sample_size: int | None = None, variant: str = "standard" ) -> list[dict]: - from datasets import load_dataset + from bench_core.datautil import load_rows - ds = load_dataset(_DATASET_REPO, _CONFIGS[variant], split=_SPLIT) - rows = list(ds)[:sample_size] if sample_size else list(ds) + rows = load_rows(_DATASET_REPO, _SPLIT, sample_size, config=_CONFIGS[variant]) samples = [] for row in rows: row = dict(row) diff --git a/benchmarks/obj_detection/bench.py b/benchmarks/obj_detection/bench.py index 37fa650..b8b7fdd 100644 --- a/benchmarks/obj_detection/bench.py +++ b/benchmarks/obj_detection/bench.py @@ -215,11 +215,10 @@ def _parse_variant(variant: str) -> tuple[str, str]: def load_samples(sample_size: int | None = None, variant: str = "val") -> list[dict]: - from datasets import load_dataset + from bench_core.datautil import load_rows dataset_id, split = _parse_variant(variant) - ds = load_dataset(dataset_id, split=split) - rows = list(ds)[:sample_size] if sample_size else list(ds) + rows = load_rows(dataset_id, split, sample_size) samples = [] for i, row in enumerate(rows): answers = row.get("answer") diff --git a/benchmarks/ocrbench_v2/bench.py b/benchmarks/ocrbench_v2/bench.py index a0a49c8..df508c0 100644 --- a/benchmarks/ocrbench_v2/bench.py +++ b/benchmarks/ocrbench_v2/bench.py @@ -86,35 +86,49 @@ def get_spotting_prompt(question: str) -> str: return TEXT_SPOTTING_PROMPT_TEMPLATE.format(level=level) +def _mk_sample(row, idx=None, image=None) -> dict: + question = row["question"] + if row["type"] == "text spotting en": + question = get_spotting_prompt(question) + s = { + "id": row["id"], + "dataset_name": row["dataset_name"], + "type": row["type"], + "question": question, + "answers": row["answers"], + } + # full runs carry only an index (image read lazily); smokes embed the image + if image is not None: + s["image"] = image + else: + s["idx"] = idx + return s + + def load_samples(sample_size: int | None = None) -> list[dict]: global _DATASET from datasets import load_dataset + if sample_size: + # stream the first N so a smoke doesn't download all 10k images + ds = load_dataset(_DATASET_ID, split=_SPLIT, streaming=True) + return [_mk_sample(dict(r), image=r["image"]) for r in ds.take(sample_size)] + + # full run: keep the split memory-mapped and read images lazily by index + # (materializing 10k decoded images would OOM) ds = load_dataset(_DATASET_ID, split=_SPLIT) _DATASET = ds meta = ds.select_columns(["id", "dataset_name", "type", "question", "answers"]) - n = len(meta) if sample_size is None else min(sample_size, len(meta)) - samples = [] - for i in range(n): - row = meta[i] - question = row["question"] - if row["type"] == "text spotting en": - question = get_spotting_prompt(question) - samples.append( - { - "id": row["id"], - "idx": i, - "dataset_name": row["dataset_name"], - "type": row["type"], - "question": question, - "answers": row["answers"], - } - ) - return samples + return [_mk_sample(meta[i], idx=i) for i in range(len(meta))] def build_request(sample: dict, mode: str) -> Request: - image = _DATASET[sample["idx"]]["image"] # decoded lazily + if "image" in sample: + image = sample["image"] # streamed smoke: embedded + elif "idx" in sample: + image = _DATASET[sample["idx"]]["image"] # full run: decoded lazily + else: + raise KeyError("OCRBench sample missing both 'image' and 'idx'") img = encode_image( image, "image/jpeg" ) # RGB + JPEG q95, no resize (matches runner) diff --git a/tests/test_datautil.py b/tests/test_datautil.py new file mode 100644 index 0000000..55c56d5 --- /dev/null +++ b/tests/test_datautil.py @@ -0,0 +1,48 @@ +import sys +import types + +from bench_core.datautil import load_rows + + +def _install_fake(monkeypatch, capture): + """Replace the `datasets` module so load_rows imports our stub.""" + + class _Stream: + def __init__(self, rows): + self.rows = rows + + def take(self, n): + return self.rows[:n] + + def fake_load_dataset(*args, **kwargs): + capture.append((args, kwargs)) + rows = [{"id": i} for i in range(100)] + return _Stream(rows) if kwargs.get("streaming") else rows + + mod = types.ModuleType("datasets") + mod.load_dataset = fake_load_dataset + monkeypatch.setitem(sys.modules, "datasets", mod) + + +def test_load_rows_streams_only_first_n_when_sampling(monkeypatch): + cap = [] + _install_fake(monkeypatch, cap) + rows = load_rows("ds/x", "test", sample_size=3) + assert rows == [{"id": 0}, {"id": 1}, {"id": 2}] + # the whole point: a smoke must not download the full split + assert cap[0][1]["streaming"] is True + + +def test_load_rows_full_load_when_no_sample(monkeypatch): + cap = [] + _install_fake(monkeypatch, cap) + rows = load_rows("ds/x", "test") + assert len(rows) == 100 + assert "streaming" not in cap[0][1] + + +def test_load_rows_passes_config_positionally(monkeypatch): + cap = [] + _install_fake(monkeypatch, cap) + load_rows("ds/x", "test", sample_size=1, config="cfg") + assert cap[0][0] == ("ds/x", "cfg") diff --git a/tests/test_results.py b/tests/test_results.py index 638c485..9fb124b 100644 --- a/tests/test_results.py +++ b/tests/test_results.py @@ -4,9 +4,22 @@ found report_scores dropping 5 of 7 RefCOCO runners by prefix mismatch). """ +import json + from bench_core.results import RunStore, discover, model_slug +def test_sampled_run_is_isolated_from_full_run(tmp_path): + """A --sample smoke (written under an isolated _smoke root) must neither + clobber a full run's metrics on disk nor appear in the leaderboard.""" + full = RunStore("gpqa", "inkling", root=tmp_path) + full.write_metrics({"n": 198}) + smoke = RunStore("gpqa", "inkling", root=tmp_path / "_smoke") + smoke.write_metrics({"n": 3}) + assert json.loads(full.metrics_path.read_text())["n"] == 198 + assert [d["n"] for d in discover(tmp_path)] == [198] + + def test_model_slug_is_canonical(): assert model_slug("accounts/fireworks/models/inkling-small") == "inkling-small" assert model_slug("google/gemini-3.7-flash") == "gemini-3.7-flash" From b6bca7fc0ed19a5c5613d96adacebfe4beefef4c Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Mon, 17 Aug 2026 08:24:59 +0530 Subject: [PATCH 29/38] perf(bench): bound full-run memory for image benchmarks (worker pool + lazy-idx) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner did asyncio.gather(process(s) for s in pending) with build_request above the first await, so every request was built up front — a full image benchmark decoded/encoded all images at once (RefCOCO val list(ds) est. ~14 GB, OOMing a 16 GB CI runner). Two changes: - runner: bounded worker pool (min(max_in_flight, len) workers pulling a queue), so build_request runs only when a worker picks up a sample -> at most max_in_flight decoded images at once. All 5 existing runner tests pass unchanged; new test asserts peak in-flight <= max_in_flight. - refcoco/mmmu_pro: full runs keep the split memory-mapped and read images lazily by idx (samples carry idx, not images); smokes embed the streamed image via a dual image/idx path in build_request. Verified RSS flat: RefCOCO val ~1.7 GB, MMMU-Pro ~0.9 GB (were climbing to OOM); live smokes pass. --- bench_core/runner.py | 44 ++++++++++----- benchmarks/mmmu_pro/bench.py | 91 +++++++++++++++++++------------ benchmarks/obj_detection/bench.py | 82 ++++++++++++++++++---------- tests/test_mmmu_pro_bench.py | 42 ++++++++++++++ tests/test_refcoco_bench.py | 22 ++++++++ tests/test_runner.py | 40 ++++++++++++++ 6 files changed, 243 insertions(+), 78 deletions(-) diff --git a/bench_core/runner.py b/bench_core/runner.py index b8cd286..90b874f 100644 --- a/bench_core/runner.py +++ b/bench_core/runner.py @@ -72,7 +72,6 @@ def done(r): pending = [s for s in samples if s[id_key] not in completed] limiter = _RateLimiter(rate_limit) - sem = asyncio.Semaphore(max_in_flight) result = RunResult() lock = asyncio.Lock() @@ -80,25 +79,40 @@ async def process(sample): req = build_request(sample) for attempt in range(max_retries + 1): await limiter.acquire() - async with sem: - try: - resp, hints = await asyncio.to_thread( - execute, adapter, client, req, caps, model_id - ) - except BenchError as be: - kind = be.classification.kind - if kind in _RETRYABLE and attempt < max_retries: - if backoff_base > 0: - await asyncio.sleep(backoff_base * (2**attempt)) - continue - await _record_failure(sample, be, store, result, lock, id_key) - return + try: + resp, hints = await asyncio.to_thread( + execute, adapter, client, req, caps, model_id + ) + except BenchError as be: + kind = be.classification.kind + if kind in _RETRYABLE and attempt < max_retries: + if backoff_base > 0: + await asyncio.sleep(backoff_base * (2**attempt)) + continue + await _record_failure(sample, be, store, result, lock, id_key) + return await _record_success( sample, resp, hints, parse, store, result, lock, id_key ) return - await asyncio.gather(*(process(s) for s in pending)) + # Bounded worker pool: build_request runs only when a worker pulls a sample, + # so at most max_in_flight requests (and their decoded images) exist at once. + # An eager gather(process(s) for s in pending) builds every request up front + # — for a full image benchmark that decodes all ~10k images and OOMs CI. + queue: asyncio.Queue = asyncio.Queue() + for s in pending: + queue.put_nowait(s) + + async def worker(): + while True: + try: + sample = queue.get_nowait() + except asyncio.QueueEmpty: + return + await process(sample) + + await asyncio.gather(*(worker() for _ in range(min(max_in_flight, len(pending))))) return result diff --git a/benchmarks/mmmu_pro/bench.py b/benchmarks/mmmu_pro/bench.py index 8c92615..4244314 100644 --- a/benchmarks/mmmu_pro/bench.py +++ b/benchmarks/mmmu_pro/bench.py @@ -19,6 +19,7 @@ _SPLIT = "test" _CONFIGS = {"standard": "standard (10 options)", "vision": "vision"} _MAX_IMAGE_SIDE = 1536 +_DATASET = None # full-run split; images read lazily by idx to bound memory _LETTERS = list("ABCDEFGHIJ") _PROMPT_STANDARD = ( @@ -63,53 +64,73 @@ def _options_block(options: list[str]) -> str: return "\n".join(f"{ltr}. {opt}" for ltr, opt in zip(_LETTERS, options)) +def _img_cols(variant: str) -> list[str]: + return ["image"] if variant == "vision" else [f"image_{i}" for i in range(1, 8)] + + +def _row_images(row, setting: str) -> list: + if setting == "vision": + return [row["image"]] + imgs = [row.get(f"image_{i}") for i in range(1, 8)] + return [im for im in imgs if im is not None] + + +def _mk_sample(row, variant: str, idx: int | None = None) -> dict: + setting = "vision" if variant == "vision" else "standard" + s = { + "id": row["id"], + "setting": setting, + "options": _options(row["options"]), + "answer": str(row["answer"]).strip().upper(), + "subject": row.get("subject"), + "topic_difficulty": None + if setting == "vision" + else row.get("topic_difficulty"), + } + if setting == "standard": + s["question"] = row["question"] + if idx is None: + s["images"] = _row_images(row, setting) # smoke embeds streamed images + else: + s["idx"] = idx # full run reads images lazily from _DATASET + return s + + def load_samples( sample_size: int | None = None, variant: str = "standard" ) -> list[dict]: - from bench_core.datautil import load_rows - - rows = load_rows(_DATASET_REPO, _SPLIT, sample_size, config=_CONFIGS[variant]) - samples = [] - for row in rows: - row = dict(row) - if variant == "vision": - samples.append( - { - "id": row["id"], - "setting": "vision", - "image": row["image"], - "options": _options(row["options"]), - "answer": str(row["answer"]).strip().upper(), - "subject": row.get("subject"), - "topic_difficulty": None, - } - ) - else: - images = [row.get(f"image_{i}") for i in range(1, 8)] - samples.append( - { - "id": row["id"], - "setting": "standard", - "question": row["question"], - "options": _options(row["options"]), - "images": [im for im in images if im is not None], - "answer": str(row["answer"]).strip().upper(), - "subject": row.get("subject"), - "topic_difficulty": row.get("topic_difficulty"), - } - ) - return samples + global _DATASET + + if sample_size: + from bench_core.datautil import load_rows + + rows = load_rows(_DATASET_REPO, _SPLIT, sample_size, config=_CONFIGS[variant]) + return [_mk_sample(dict(r), variant) for r in rows] + + # full run: keep the split memory-mapped and read images lazily by idx. + # Build samples from an image-free view so the load pass doesn't decode + # every image (materializing all ~1730 rows of images OOMs CI). + from datasets import load_dataset + + _DATASET = load_dataset(_DATASET_REPO, _CONFIGS[variant], split=_SPLIT) + present = [c for c in _img_cols(variant) if c in _DATASET.column_names] + meta = _DATASET.remove_columns(present) + return [_mk_sample(meta[i], variant, idx=i) for i in range(len(meta))] def build_request(sample: dict, mode: str) -> Request: + if "images" in sample: + pil_images = sample["images"] # streamed smoke: embedded + elif "idx" in sample: + pil_images = _row_images(_DATASET[sample["idx"]], sample["setting"]) # lazy + else: + raise KeyError("MMMU-Pro sample missing both 'images' and 'idx'") if sample["setting"] == "vision": prompt = _PROMPT_VISION - pil_images = [sample["image"]] else: prompt = _PROMPT_STANDARD.format( question=sample["question"], options=_options_block(sample["options"]) ) - pil_images = sample["images"] parts = [TextPart(prompt)] parts += [ encode_image(im, "image/jpeg", max_side=_MAX_IMAGE_SIDE) for im in pil_images diff --git a/benchmarks/obj_detection/bench.py b/benchmarks/obj_detection/bench.py index b8b7fdd..e9b78a3 100644 --- a/benchmarks/obj_detection/bench.py +++ b/benchmarks/obj_detection/bench.py @@ -37,6 +37,7 @@ "g": "lmms-lab/RefCOCOg", } _MAX_SIDE = 1024 +_DATASET = None # full-run split; images read lazily by idx to bound memory IOU_THRESHOLD = 0.5 _THRESHOLDS = [0.3, 0.5, 0.7, 0.75, 0.9] @@ -214,42 +215,67 @@ def _parse_variant(variant: str) -> tuple[str, str]: return _DATASETS[""], variant -def load_samples(sample_size: int | None = None, variant: str = "val") -> list[dict]: - from bench_core.datautil import load_rows +def _mk_sample(row, i, image, idx=None) -> dict | None: + answers = row.get("answer") + if isinstance(answers, str): + exprs = [answers] + elif isinstance(answers, list): + exprs = [str(a) for a in answers if str(a).strip()] + else: + exprs = [] + if not exprs: + return None + ow, oh = image.size # header-only; not retained on the full path + sw, sh = _sent_dims(ow, oh) + gt = coco_bbox_to_xyxy(list(row["bbox"]), sw / ow, sh / oh) + s = { + "id": f"{row.get('question_id', i)}_{i}", + "expression": exprs[0], + "sent_w": sw, + "sent_h": sh, + "gt_bbox_xyxy": gt, + } + if idx is None: + s["image"] = image # smoke embeds the streamed image + else: + s["idx"] = idx # full run reads it lazily from _DATASET + return s + +def load_samples(sample_size: int | None = None, variant: str = "val") -> list[dict]: + global _DATASET dataset_id, split = _parse_variant(variant) - rows = load_rows(dataset_id, split, sample_size) + + if sample_size: + from bench_core.datautil import load_rows + + rows = load_rows(dataset_id, split, sample_size) + out = (_mk_sample(r, i, r["image"]) for i, r in enumerate(rows)) + return [s for s in out if s] + + # full run: keep the split memory-mapped and read images lazily by idx + # (list(ds) materializes all ~8.8k decoded images and OOMs CI). + from datasets import load_dataset + + _DATASET = load_dataset(dataset_id, split=split) samples = [] - for i, row in enumerate(rows): - answers = row.get("answer") - if isinstance(answers, str): - exprs = [answers] - elif isinstance(answers, list): - exprs = [str(a) for a in answers if str(a).strip()] - else: - exprs = [] - if not exprs: - continue - image = row["image"] - ow, oh = image.size - sw, sh = _sent_dims(ow, oh) - gt = coco_bbox_to_xyxy(list(row["bbox"]), sw / ow, sh / oh) - samples.append( - { - "id": f"{row.get('question_id', i)}_{i}", - "expression": exprs[0], - "image": image, - "sent_w": sw, - "sent_h": sh, - "gt_bbox_xyxy": gt, - } - ) + for i in range(len(_DATASET)): + row = _DATASET[i] + s = _mk_sample(row, i, row["image"], idx=i) + if s: + samples.append(s) return samples def build_request(sample: dict, mode: str) -> Request: prompt = _PROMPT.format(expression=sample["expression"]) - img = encode_image(sample["image"], "image/jpeg", max_side=_MAX_SIDE) + if "image" in sample: + image = sample["image"] # streamed smoke: embedded + elif "idx" in sample: + image = _DATASET[sample["idx"]]["image"] # full run: decoded lazily + else: + raise KeyError("RefCOCO sample missing both 'image' and 'idx'") + img = encode_image(image, "image/jpeg", max_side=_MAX_SIDE) return Request( [Message("user", [TextPart(prompt), img])], ReasoningSpec(mode), temperature=0.0 ) diff --git a/tests/test_mmmu_pro_bench.py b/tests/test_mmmu_pro_bench.py index ee93950..802b1d7 100644 --- a/tests/test_mmmu_pro_bench.py +++ b/tests/test_mmmu_pro_bench.py @@ -18,6 +18,48 @@ ] +def test_build_request_embedded_or_lazy_idx(monkeypatch): + """Smokes embed images; full runs carry an index and read images lazily + from _DATASET (so the samples list doesn't hold ~1730 rows of images).""" + from PIL import Image + + img = Image.new("RGB", (16, 16)) + + # standard, embedded (smoke): text + 2 image parts + s = { + "id": "x", + "setting": "standard", + "question": "q?", + "options": ["a", "b"], + "answer": "A", + "images": [img, img], + } + assert len(bench.build_request(s, "off").messages[0].parts) == 3 + + # standard, lazy idx: images read from the _DATASET row (image_2 is None) + monkeypatch.setattr(bench, "_DATASET", {5: {"image_1": img, "image_2": None}}) + s2 = { + "id": "y", + "setting": "standard", + "question": "q?", + "options": ["a", "b"], + "answer": "B", + "idx": 5, + } + assert len(bench.build_request(s2, "off").messages[0].parts) == 2 + + # vision, lazy idx + monkeypatch.setattr(bench, "_DATASET", {7: {"image": img}}) + s3 = { + "id": "z", + "setting": "vision", + "options": ["a", "b"], + "answer": "C", + "idx": 7, + } + assert len(bench.build_request(s3, "off").messages[0].parts) == 2 + + def test_parse_answer_a_to_j(): assert bench.parse_answer("H") == "H" assert bench.parse_answer("(J)") == "J" diff --git a/tests/test_refcoco_bench.py b/tests/test_refcoco_bench.py index c0032c1..971fbce 100644 --- a/tests/test_refcoco_bench.py +++ b/tests/test_refcoco_bench.py @@ -27,6 +27,28 @@ def test_parse_variant_restores_plus_and_g_datasets(): assert bench._parse_variant("g-test") == ("lmms-lab/RefCOCOg", "test") +def test_build_request_embedded_image_or_lazy_idx(monkeypatch): + """Smokes embed the image; full runs carry an index and read the image + lazily from _DATASET (so the samples list doesn't hold all 8.8k images).""" + from PIL import Image + + img = Image.new("RGB", (16, 16)) + base = { + "id": "x", + "expression": "cat", + "sent_w": 16, + "sent_h": 16, + "gt_bbox_xyxy": [0, 0, 1, 1], + } + + req = bench.build_request({**base, "image": img}, "off") + assert len(req.messages[0].parts) == 2 # text + image + + monkeypatch.setattr(bench, "_DATASET", {5: {"image": img}}) + req2 = bench.build_request({**base, "idx": 5}, "off") + assert len(req2.messages[0].parts) == 2 + + def test_compute_iou(): assert bench.compute_iou([0, 0, 10, 10], [0, 0, 10, 10]) == 1.0 assert bench.compute_iou([0, 0, 10, 10], [20, 20, 30, 30]) == 0.0 diff --git a/tests/test_runner.py b/tests/test_runner.py index 6ca6e42..74c05c7 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -111,6 +111,46 @@ async def test_fatal_error_records_failure_without_retry(tmp_path): assert store.load_responses()[0]["response"] is None +async def test_build_request_is_bounded_by_max_in_flight(tmp_path): + """Requests must be built lazily as workers pick up samples — not all up + front. Otherwise a full image benchmark decodes/encodes every image at once + and OOMs CI. inflight is incremented in build_request and decremented in + parse; all samples succeed here, so the pairing holds (parse runs only on + success).""" + ids = [str(i) for i in range(10)] + adapter = MapAdapter({i: ["ok"] for i in ids}) + store = RunStore("t", "m", root=tmp_path) + inflight = peak = 0 + + def build_request(sample): + nonlocal inflight, peak + inflight += 1 + peak = max(peak, inflight) + return Request( + [Message("user", [TextPart(sample["id"])])], ReasoningSpec("off") + ) + + def parse(resp, sample): + nonlocal inflight + inflight -= 1 + return resp.text + + await run_benchmark( + adapter=adapter, + client=None, + caps=CAPS, + model_id="m", + samples=[{"id": i} for i in ids], + build_request=build_request, + parse=parse, + store=store, + backoff_base=0.0, + rate_limit=0, + max_in_flight=2, + ) + assert peak <= 2 # not 10: at most max_in_flight requests built at a time + + async def test_capability_hints_are_aggregated(tmp_path): adapter = MapAdapter( {"a": [Exception("Thinking level MINIMAL is not supported"), "ok"]} From cd024c9c71e40d8fd7f8c52c73a63e6d4651fb83 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Mon, 17 Aug 2026 08:27:39 +0530 Subject: [PATCH 30/38] chore: delete dead provider shims + demo entrypoint src/commons_{openai,anthropic,fireworks,openrouter,gemini}.py had 0 importers (superseded by bench_core/providers); main.py + src/commons.py were a hello-world Interfaze demo referenced nowhere. Kept src/commons_reducto.py (olmocr reducto). --- main.py | 15 --------- src/commons.py | 61 ------------------------------------ src/commons_anthropic.py | 44 -------------------------- src/commons_fireworks.py | 66 --------------------------------------- src/commons_gemini.py | 44 -------------------------- src/commons_openai.py | 44 -------------------------- src/commons_openrouter.py | 46 --------------------------- 7 files changed, 320 deletions(-) delete mode 100644 main.py delete mode 100644 src/commons.py delete mode 100644 src/commons_anthropic.py delete mode 100644 src/commons_fireworks.py delete mode 100644 src/commons_gemini.py delete mode 100644 src/commons_openai.py delete mode 100644 src/commons_openrouter.py diff --git a/main.py b/main.py deleted file mode 100644 index cce44f8..0000000 --- a/main.py +++ /dev/null @@ -1,15 +0,0 @@ -# main.py -from src.commons import invoke_interfaze -from pydantic import BaseModel, Field - - -class ResponseModel(BaseModel): - capital: str = Field(..., description="The capital of the country", required=True) - - -response = invoke_interfaze( - messages=[{"role": "user", "content": "What is the capital of France?"}], - structured_response=True, - structure_definition=ResponseModel, -) -print(response.choices[0].message.content) diff --git a/src/commons.py b/src/commons.py deleted file mode 100644 index e18cabc..0000000 --- a/src/commons.py +++ /dev/null @@ -1,61 +0,0 @@ -from pydantic import BaseModel -from openai import OpenAI -import os - -from dotenv import load_dotenv - -load_dotenv() - - -INTERFAZE_API_KEY = os.getenv("INTERFAZE_API_KEY", None) - -INTERFAZE_BASE_URL = os.getenv("OPENAI_BASE_URL", "https://api.interfaze.ai/v1") - -# Built only when the key is present. The multi-provider runners import shared -# helpers from the interfaze scripts, so a hard raise here would block running -# any other provider; the error surfaces on first use instead. -interfaze_client = ( - OpenAI(base_url=INTERFAZE_BASE_URL, api_key=INTERFAZE_API_KEY) - if INTERFAZE_API_KEY - else None -) - - -def invoke_interfaze( - messages: list[dict], - model: str = "interfaze-beta", - stream: bool = False, - structured_response: bool = False, - structure_definition: BaseModel | None = None, - reasoning_effort: str | None = None, - temperature: float | None = None, -) -> dict: - """For invoking interfaze with images you can use - messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is in this image?", "image_url": {image_url: {"url": "https://example.com/image.jpg"}}}, - ] - - 2. For structured response you can define a pydantic model and pass it as structure_definition argument. For example: - class ResponseModel(BaseModel): - capital: str - Then you can invoke the function as follows: - response = invoke_interfaze( - messages=[{"role": "user", "content": "What is the capital of France?"}], - structured_response=True, - structure_definition=ResponseModel - ) - """ - if interfaze_client is None: - raise ValueError( - "INTERFAZE_API_KEY is not set in environment variables get it from https://interfaze.ai/dashboard" - ) - kwargs: dict = {"model": model, "messages": messages, "stream": stream} - if reasoning_effort is not None: - kwargs["reasoning_effort"] = reasoning_effort - if temperature is not None: - kwargs["temperature"] = temperature - try: - return interfaze_client.chat.completions.create(**kwargs) - except Exception as e: - raise RuntimeError(f"Error invoking Interfaze API: {e}") from e diff --git a/src/commons_anthropic.py b/src/commons_anthropic.py deleted file mode 100644 index 7c2bd4a..0000000 --- a/src/commons_anthropic.py +++ /dev/null @@ -1,44 +0,0 @@ -import os - -import anthropic -from dotenv import load_dotenv - -load_dotenv() - - -if (ANTHROPIC_API_KEY := os.getenv("ANTHROPIC_API_KEY", None)) is None: - raise ValueError( - "ANTHROPIC_API_KEY is not set in environment variables get it from https://console.anthropic.com/" - ) - -anthropic_client = anthropic.Anthropic(api_key=ANTHROPIC_API_KEY) - - -def invoke_anthropic( - messages: list[dict], - model: str = "claude-sonnet-4-6", - max_tokens: int = 4096, - system: str | None = None, -): - """Invoke Anthropic Messages API with thinking disabled (omitted). - - Example image message: - messages = [{ - "role": "user", - "content": [ - {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": b64_str}}, - {"type": "text", "text": "What is in this image?"}, - ], - }] - """ - kwargs: dict = { - "model": model, - "max_tokens": max_tokens, - "messages": messages, - } - if system is not None: - kwargs["system"] = system - try: - return anthropic_client.messages.create(**kwargs) - except Exception as e: - raise RuntimeError(f"Error invoking Anthropic API: {e}") from e diff --git a/src/commons_fireworks.py b/src/commons_fireworks.py deleted file mode 100644 index e62f0d1..0000000 --- a/src/commons_fireworks.py +++ /dev/null @@ -1,66 +0,0 @@ -import os - -from openai import OpenAI -from dotenv import load_dotenv - -load_dotenv() - -# The project .env may spell this key lowercase (`fireworks_api_key`); -# load_dotenv() copies the name verbatim, so accept either casing. -FIREWORKS_API_KEY = os.getenv("FIREWORKS_API_KEY") or os.getenv("fireworks_api_key") - -if FIREWORKS_API_KEY is None: - raise ValueError( - "FIREWORKS_API_KEY is not set in environment variables get it from https://app.fireworks.ai/settings/users/api-keys" - ) - -FIREWORKS_BASE_URL = os.getenv( - "FIREWORKS_BASE_URL", "https://api.fireworks.ai/inference/v1" -) - -# Inkling (Thinking Machines) — text+image+audio in, text out, 1M context. -# inkling-small is NOT on Fireworks serverless; it needs an on-demand -# dedicated deployment, after which its model id is the same string. -INKLING = "accounts/fireworks/models/inkling" -INKLING_SMALL = "accounts/fireworks/models/inkling-small" - -# Inkling always thinks. reasoning_effort accepts none|low|medium|high|xhigh|max -# — "none" is the floor (still emits reasoning tokens), there is no true off. -REASONING_FLOOR = "none" -REASONING_MAX = "high" - -fireworks_client = OpenAI(api_key=FIREWORKS_API_KEY, base_url=FIREWORKS_BASE_URL) - - -def invoke_fireworks( - messages: list[dict], - model: str = INKLING, - reasoning_effort: str | None = None, - temperature: float | None = None, - max_tokens: int | None = None, - stream: bool = False, -): - """Invoke a Fireworks-hosted model over the OpenAI-compatible endpoint. - - Image input uses the standard `image_url` block with a data URL. Audio - input uses an `audio_url` block with a data URL (Fireworks rejects - OpenAI's `input_audio` shape for this model): - messages = [{ - "role": "user", - "content": [ - {"type": "text", "text": "Transcribe this audio."}, - {"type": "audio_url", "audio_url": {"url": "data:audio/wav;base64,..."}}, - ], - }] - """ - kwargs: dict = {"model": model, "messages": messages, "stream": stream} - if reasoning_effort is not None: - kwargs["reasoning_effort"] = reasoning_effort - if temperature is not None: - kwargs["temperature"] = temperature - if max_tokens is not None: - kwargs["max_tokens"] = max_tokens - try: - return fireworks_client.chat.completions.create(**kwargs) - except Exception as e: - raise RuntimeError(f"Error invoking Fireworks API: {e}") from e diff --git a/src/commons_gemini.py b/src/commons_gemini.py deleted file mode 100644 index 9885b7f..0000000 --- a/src/commons_gemini.py +++ /dev/null @@ -1,44 +0,0 @@ -import os - -from google import genai -from google.genai import types -from dotenv import load_dotenv - -load_dotenv() - - -if (GEMINI_API_KEY := os.getenv("GEMINI_API_KEY", None)) is None: - raise ValueError( - "GEMINI_API_KEY is not set in environment variables get it from https://aistudio.google.com/apikey" - ) - -gemini_client = genai.Client(api_key=GEMINI_API_KEY) - - -def invoke_gemini( - contents, - model: str = "gemini-3.1-pro-preview", - system: str | None = None, -): - """Invoke Gemini generate_content. - - Note: Gemini 3.x Pro models require thinking mode (reject thinking_budget=0). - We let the API use its default thinking level rather than forcing it off. - If you switch to a Flash variant and want thinking disabled, add - `thinking_config=types.ThinkingConfig(thinking_budget=0)` to the config below. - - `contents` can be a list mixing strings and PIL.Image / types.Part objects, e.g.: - contents = [question_text, pil_image] - """ - config = types.GenerateContentConfig( - system_instruction=system, - thinking_config=types.ThinkingConfig(thinking_budget=0), - ) - try: - return gemini_client.models.generate_content( - model=model, - contents=contents, - config=config, - ) - except Exception as e: - raise RuntimeError(f"Error invoking Gemini API: {e}") from e diff --git a/src/commons_openai.py b/src/commons_openai.py deleted file mode 100644 index 1b823f1..0000000 --- a/src/commons_openai.py +++ /dev/null @@ -1,44 +0,0 @@ -import os - -from openai import OpenAI -from dotenv import load_dotenv - -load_dotenv() - - -if (OPENAI_API_KEY := os.getenv("OPENAI_API_KEY", None)) is None: - raise ValueError( - "OPENAI_API_KEY is not set in environment variables get it from https://platform.openai.com/api-keys" - ) - -# Explicit so we don't inherit OPENAI_BASE_URL from commons.py (Interfaze). -# Override with OPENAI_API_BASE_URL for Azure, proxies, or compatible endpoints. -OPENAI_BASE_URL = os.getenv("OPENAI_API_BASE_URL", "https://api.openai.com/v1") - -openai_client = OpenAI(api_key=OPENAI_API_KEY, base_url=OPENAI_BASE_URL) - - -def invoke_openai( - messages: list[dict], - model: str = "gpt-5.4", - stream: bool = False, -): - """Invoke OpenAI Chat Completions API (non-reasoning model, no thinking). - - Example image message: - messages = [{ - "role": "user", - "content": [ - {"type": "text", "text": "What is in this image?"}, - {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,..."}}, - ], - }] - """ - try: - return openai_client.chat.completions.create( - model=model, - messages=messages, - stream=stream, - ) - except Exception as e: - raise RuntimeError(f"Error invoking OpenAI API: {e}") from e diff --git a/src/commons_openrouter.py b/src/commons_openrouter.py deleted file mode 100644 index 4d0452d..0000000 --- a/src/commons_openrouter.py +++ /dev/null @@ -1,46 +0,0 @@ -import os - -from openai import OpenAI -from dotenv import load_dotenv - -load_dotenv() - - -# This project's .env spells the key lowercase; accept either casing. -if ( - OPENROUTER_API_KEY := os.getenv("OPENROUTER_API_KEY") - or os.getenv("openrouter_api_key") -) is None: - raise ValueError( - "OPENROUTER_API_KEY is not set in environment variables get it from https://openrouter.ai/keys" - ) - -OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1" - -# Explicit timeout + retries: without them a single hung connection blocks -# forever. Observed on Spider2 — one request never returned and stalled the -# whole benchmark queue for 5h at 0% CPU with 134/135 already done. -openrouter_client = OpenAI( - api_key=OPENROUTER_API_KEY, - base_url=OPENROUTER_BASE_URL, - timeout=180.0, - max_retries=2, -) - - -def invoke_openrouter( - messages: list[dict], - model: str, - temperature: float | None = None, - extra_body: dict | None = None, -): - """Invoke any OpenRouter-hosted model via the OpenAI-compatible chat-completions API.""" - kwargs: dict = {"model": model, "messages": messages} - if temperature is not None: - kwargs["temperature"] = temperature - if extra_body is not None: - kwargs["extra_body"] = extra_body - try: - return openrouter_client.chat.completions.create(**kwargs) - except Exception as e: - raise RuntimeError(f"Error invoking OpenRouter API: {e}") from e From bfc5a62c94a9e2171043950e57524754e947a9a6 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Mon, 17 Aug 2026 08:28:37 +0530 Subject: [PATCH 31/38] fix(cli): exit hard after a run so google-genai threads don't hang CI A gemini run finished and wrote all results but the process never exited (the google-genai SDK leaves non-daemon threads), so a CI job would hang until its timeout. __main__ now flushes and os._exit(0) after main() returns; a SystemExit from an error still propagates first with its code. Verified: gemini run now exits 0 on its own in ~4s instead of hanging. --- bench_core/__main__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/bench_core/__main__.py b/bench_core/__main__.py index 08837ce..722410f 100644 --- a/bench_core/__main__.py +++ b/bench_core/__main__.py @@ -1,3 +1,14 @@ +import os +import sys + from bench_core.cli import main main() +# Some provider SDKs (google-genai) leave non-daemon background threads that keep +# the process alive after the run finishes and every result is already fsync'd. +# This is a one-shot CLI with nothing left to do, so exit hard rather than hang a +# CI job until its timeout. A SystemExit from an error propagates before this and +# keeps its non-zero code. +sys.stdout.flush() +sys.stderr.flush() +os._exit(0) From 362dc6362d622573974fe69693759da6d2cf49e8 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Mon, 17 Aug 2026 08:32:27 +0530 Subject: [PATCH 32/38] chore: drop the one-time migrate-results backfill The backfill has done its job (results now live in the contract). Removes bench_core/migrate.py, tests/test_migrate.py, the migrate-results CLI subcommand, and stale doc references. --- .gitignore | 2 +- bench_core/cli.py | 17 ---- bench_core/migrate.py | 173 --------------------------------------- scripts/report_scores.py | 12 ++- tests/test_migrate.py | 96 ---------------------- 5 files changed, 6 insertions(+), 294 deletions(-) delete mode 100644 bench_core/migrate.py delete mode 100644 tests/test_migrate.py diff --git a/.gitignore b/.gitignore index bfbac09..98374b7 100644 --- a/.gitignore +++ b/.gitignore @@ -215,7 +215,7 @@ benchmarks/olmocr/bench/full_data/ # Benchmark outputs are never tracked — they are run artifacts, not source, and # large (hundreds of MB incl. Spider2/OCRBench responses). Kept on disk locally; -# regenerate with a run or `bench migrate-results`. +# regenerate with a run. results/ # OCRBench spotting-eval scratch (generated during scoring) diff --git a/bench_core/cli.py b/bench_core/cli.py index a7f3230..3b38aad 100644 --- a/bench_core/cli.py +++ b/bench_core/cli.py @@ -134,15 +134,6 @@ def cmd_list_targets(args) -> None: print(f"{name:22} {t.provider:11} {t.model_id}") -def cmd_migrate_results(args) -> None: - from bench_core.migrate import migrate_results - - log = migrate_results(dry_run=args.dry_run) - for line in log: - print(line) - print(f"\n{'[dry-run] ' if args.dry_run else ''}{len(log)} action(s)") - - def main(argv=None) -> None: # pick up provider keys from a repo-root .env (adapters read os.getenv) try: @@ -174,14 +165,6 @@ def main(argv=None) -> None: lt = sub.add_parser("list-targets", help="list configured targets") lt.set_defaults(func=cmd_list_targets) - mr = sub.add_parser( - "migrate-results", help="backfill legacy flat results into the new contract" - ) - mr.add_argument( - "--dry-run", action="store_true", help="show actions without writing" - ) - mr.set_defaults(func=cmd_migrate_results) - args = p.parse_args(argv) args.func(args) diff --git a/bench_core/migrate.py b/bench_core/migrate.py deleted file mode 100644 index 1138810..0000000 --- a/bench_core/migrate.py +++ /dev/null @@ -1,173 +0,0 @@ -"""One-time backfill: fold the legacy flat results/*_metrics.json (+ their -responses/predictions siblings) into the new results/// -contract, so the report reads one layout and a partial run resumes rather than -re-bill. - -The new contract keys by (benchmark, target) only — it does not encode host or -reasoning — so when several legacy runs map to the same slot (a 2-sample smoke -vs a full run, or reasoning off vs high) the one with the most *evaluated* -samples wins and the rest are logged as shadowed. Idempotent; no silent drops. -""" - -from __future__ import annotations - -import json -from collections import defaultdict -from pathlib import Path - -from bench_core.results import DEFAULT_ROOT, RunStore, model_slug - -# count of samples actually evaluated (NOT a fixed denominator like Spider2's -# total_local_subset, which is 135 even for a 1-sample smoke) -_N_KEYS = ("num_samples", "total", "total_evaluated", "n") - - -def _classify(name: str) -> str | None: - if name.startswith("ocrbench_v2_"): - return "ocrbench_v2" - if name.startswith("mmmupro_standard_"): - return "mmmu_pro_standard" - if name.startswith("mmmupro_vision_"): - return "mmmu_pro_vision" - if name.startswith("mmmlulite_"): - return "mmmlu_lite" - if name.startswith("mmmlufull_"): - return "mmmlu_full" - if name.startswith("refcoco_"): - return f"refcoco_{name.split('_')[1]}" - if name.startswith("voxpopuli_aa_"): - return "voxpopuli_aa" - if name.startswith("spider2_lite_local_"): - return "spider2_lite" - if "gpqa_diamond" in name: - return "gpqa" - return None - - -def _n(d: dict) -> int: - for k in _N_KEYS: - v = d.get(k) - if isinstance(v, int): - return v - return 0 - - -def _resp_shape(benchmark: str): - if benchmark == "spider2_lite": - return "instance_id", "pred_sql" - if benchmark.startswith("refcoco"): - return "id", "pred_bbox_xyxy" - return "id", "prediction" - - -def _migrate_responses( - metrics_path: Path, benchmark: str, store: RunStore -) -> str | None: - if benchmark == "ocrbench_v2": - src = metrics_path.with_name( - metrics_path.name.replace("_metrics.json", "_predictions.json") - ) - if not src.exists(): - return None - records = [ - { - "id": r["id"], - "response": r.get("predict", ""), - "prediction": r.get("predict", ""), - } - for r in json.loads(src.read_text()) - ] - else: - src = metrics_path.with_name( - metrics_path.name.replace("_metrics.json", "_responses.jsonl") - ) - if not src.exists(): - return None - id_key, pred_field = _resp_shape(benchmark) - records = [] - for line in src.read_text().splitlines(): - if line.strip(): - r = json.loads(line) - records.append( - { - id_key: r[id_key], - "response": r.get("response"), - "prediction": r.get(pred_field), - } - ) - # Only genuine answers migrate; empty/None responses are left to resume (a - # blank prediction must re-run, not count as done). - records = [r for r in records if (r.get("response") or "") != ""] - store._ensure_dir() - store.responses_path.write_text( - "".join(json.dumps(r, ensure_ascii=False) + "\n" for r in records) - ) - return f" responses: {src.name} ({len(records)} rows)" - - -def migrate_results( - results_dir: Path | str = DEFAULT_ROOT, dry_run: bool = False -) -> list[str]: - results_dir = Path(results_dir) - log: list[str] = [] - - groups: dict[tuple, list] = defaultdict(list) - for p in sorted(results_dir.glob("*_metrics.json")): - name = p.name - if "_oracle_" in name: # folded into the strict refcoco metrics - continue - benchmark = _classify(name) - if benchmark is None: - log.append(f"SKIP (unrecognized): {name}") - continue - d = json.loads(p.read_text()) - target = model_slug(d.get("model") or name) - groups[(benchmark, target)].append((p, d, _n(d))) - - for (benchmark, target), items in sorted(groups.items()): - winner = items[0] - for c in items[1:]: - if c[2] >= winner[2]: # tie -> later (sorted) wins, matching the old report - winner = c - p, d, n = winner - - newm = dict(d) - newm.update( - { - "benchmark": benchmark, - "target": target, - "provider": d.get("provider"), - "n": n, - "reasoning": {"mode": "high" if "high" in p.name else "off"}, - } - ) - if benchmark.startswith("refcoco"): - oracle_p = p.with_name( - p.name.replace("_metrics.json", "_oracle_metrics.json") - ) - if oracle_p.exists(): - od = json.loads(oracle_p.read_text()) - newm["oracle"] = { - "accuracy": od.get("accuracy"), - "mean_iou": od.get("mean_iou"), - "total": od.get("total"), - "interpretation_counts": od.get("interpretation_counts"), - } - - store = RunStore(benchmark, target, root=results_dir) - log.append( - f"metrics: {p.name} (n={n}) -> {benchmark}/{model_slug(target)}/metrics.json" - ) - if not dry_run: - store.write_metrics(newm) - r = _migrate_responses(p, benchmark, store) - if r: - log.append(r) - - for c in items: - if c is not winner: - log.append( - f" shadowed: {c[0].name} (n={c[2]}) — slot taken by the n={n} run" - ) - - return log diff --git a/scripts/report_scores.py b/scripts/report_scores.py index 12546c5..6bb0e2b 100644 --- a/scripts/report_scores.py +++ b/scripts/report_scores.py @@ -7,12 +7,10 @@ uv run python scripts/report_scores.py --list # just the model names uv run python scripts/report_scores.py --model inkling --tsv # paste into Sheets -Reads the results///metrics.json contract (written by the CLI -and `bench migrate-results`) plus the olmOCR run logs (olmOCR-bench prints its -per-split table to stdout instead of persisting metrics, so the logs are the only -source for those numbers). One row per target; the contract keys by -(benchmark, target), so `bench migrate-results` already picked the best run per -slot when several legacy runs collided. +Reads the results///metrics.json contract (written by the CLI) +plus the olmOCR run logs (olmOCR-bench prints its per-split table to stdout +instead of persisting metrics, so the logs are the only source for those +numbers). One row per target; the contract keys by (benchmark, target). """ from __future__ import annotations @@ -172,7 +170,7 @@ def _pct(x): def load_metrics() -> dict: """{model: {benchmark_key: payload}} from the results/// - metrics.json contract (written by the CLI and `bench migrate-results`).""" + metrics.json contract (written by the CLI).""" from bench_core.results import discover out: dict[str, dict] = {} diff --git a/tests/test_migrate.py b/tests/test_migrate.py deleted file mode 100644 index 65dd949..0000000 --- a/tests/test_migrate.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Backfill: legacy flat results -> new contract, with (benchmark, target) -dedup (full run beats a smoke) and refcoco oracle merged as a submetric. -""" - -import json - -from bench_core.migrate import migrate_results - - -def _write(root, name, obj): - (root / name).write_text(json.dumps(obj)) - - -def test_dedup_full_run_beats_smoke(tmp_path): - _write( - tmp_path, - "fireworks_inkling_thinkingoff_gpqa_diamond_metrics.json", - {"model": "accounts/fireworks/models/inkling", "accuracy": 0.87, "total": 198}, - ) - ( - tmp_path / "fireworks_inkling_thinkingoff_gpqa_diamond_responses.jsonl" - ).write_text(json.dumps({"id": "a", "response": "A", "prediction": "A"}) + "\n") - # a 2-sample smoke of the SAME model -> same (gpqa, inkling) slot - _write( - tmp_path, - "smoke_inkling_gpqa_diamond_metrics.json", - {"model": "accounts/fireworks/models/inkling", "accuracy": 0.5, "total": 2}, - ) - - log = migrate_results(tmp_path) - metrics = json.loads((tmp_path / "gpqa" / "inkling" / "metrics.json").read_text()) - assert metrics["benchmark"] == "gpqa" - assert metrics["target"] == "inkling" - assert metrics["accuracy"] == 0.87 # full run won, not the smoke - assert metrics["n"] == 198 - assert any("shadowed" in line and "n=2" in line for line in log) - - -def test_refcoco_oracle_merged_as_submetric(tmp_path): - _write( - tmp_path, - "refcoco_val_fireworks_x_metrics.json", - { - "model": "accounts/fireworks/models/inkling", - "accuracy": 0.32, - "mean_iou": 0.3, - "total": 10, - }, - ) - _write( - tmp_path, - "refcoco_val_fireworks_x_oracle_metrics.json", - {"accuracy": 0.81, "mean_iou": 0.7, "total": 10}, - ) - (tmp_path / "refcoco_val_fireworks_x_responses.jsonl").write_text( - json.dumps({"id": "1", "response": "[1,2,3,4]", "pred_bbox_xyxy": [1, 2, 3, 4]}) - + "\n" - ) - - migrate_results(tmp_path) - m = json.loads((tmp_path / "refcoco_val" / "inkling" / "metrics.json").read_text()) - assert m["accuracy"] == 0.32 - assert m["oracle"]["accuracy"] == 0.81 - # response migrated with the new prediction field for resume/scoring - rows = [ - json.loads(x) - for x in (tmp_path / "refcoco_val" / "inkling" / "responses.jsonl") - .read_text() - .splitlines() - ] - assert rows[0]["prediction"] == [1, 2, 3, 4] - - -def test_empty_responses_not_migrated(tmp_path): - _write( - tmp_path, - "voxpopuli_aa_x_metrics.json", - { - "model": "accounts/fireworks/models/inkling", - "corpus_wer": 0.1, - "num_samples": 2, - }, - ) - (tmp_path / "voxpopuli_aa_x_responses.jsonl").write_text( - json.dumps({"id": "1", "response": "hello", "prediction": "hello"}) - + "\n" - + json.dumps({"id": "2", "response": "", "prediction": ""}) - + "\n" - ) - migrate_results(tmp_path) - rows = ( - (tmp_path / "voxpopuli_aa" / "inkling" / "responses.jsonl") - .read_text() - .splitlines() - ) - assert len(rows) == 1 # the empty-response row is left to resume From 74859af6703b065ae218a7c54bf935edf09809e8 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Mon, 17 Aug 2026 08:41:34 +0530 Subject: [PATCH 33/38] feat(bench): ordered cross-provider failover with per-row host recording MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A target may declare 'fallbacks: [{provider, model_id, capabilities?}]'. The runner now takes an ordered list of routes (primary + fallbacks); on a terminal failure it advances to the next route, but ONLY when the host itself won't serve (FATAL / rate-limit / transient with retries exhausted) — NOT on EMPTY_CONTENT, since the backup runs the same weights and would just double the bill. The serving provider is recorded per row ('host') and aggregated into metrics.hosts; run.json lists the routes actually available. A proprietary target declares no fallbacks -> single route -> stops on failure (as intended). Keyless routes are skipped loudly (error only if none remain). Resume accepts mixed provenance across hosts (the host field is the audit trail). execute() never mutates the caller's req/caps (dataclasses.replace), so a route inherits nothing from a prior route. Example: inkling-small = OpenRouter -> Fireworks. 185 tests (7 new); teeth-checked that EMPTY_CONTENT does not fail over. --- bench_core/cli.py | 40 +++++++++++------- bench_core/config.py | 47 +++++++++++++++++++++ bench_core/runner.py | 78 ++++++++++++++++++++++++---------- bench_core/targets.yaml | 5 +++ tests/test_config.py | 53 ++++++++++++++++++++++++ tests/test_runner.py | 92 +++++++++++++++++++++++++++++++++++------ 6 files changed, 266 insertions(+), 49 deletions(-) diff --git a/bench_core/cli.py b/bench_core/cli.py index 3b38aad..6103390 100644 --- a/bench_core/cli.py +++ b/bench_core/cli.py @@ -6,10 +6,9 @@ import inspect from bench_core.config import ( - build_adapter, + build_routes, load_all_targets, load_target, - resolve_capabilities, ) from bench_core.results import RunStore from bench_core.runner import run_benchmark @@ -37,13 +36,19 @@ def cmd_run(args) -> None: target = load_target(args.target) bench = _import_benchmark(args.benchmark) mode = args.reasoning or bench.DEFAULTS.get("reasoning", "off") - caps = resolve_capabilities(target, benchmark=args.benchmark) - adapter = build_adapter(target) - if not adapter.resolve_key(): - raise SystemExit( - f"no API key for provider {target.provider!r} (looked for {adapter.key_spec})" - ) - client = adapter.build_client() + # primary provider + ordered fallbacks; keep only routes whose key is present + routes = build_routes(target, benchmark=args.benchmark) + live = [] + for rt in routes: + if rt.adapter.resolve_key(): + rt.client = rt.adapter.build_client() + live.append(rt) + else: + print( + f"skipping route {rt.provider} (no API key; looked for {rt.adapter.key_spec})" + ) + if not live: + raise SystemExit(f"no API key for any route of target {target.name!r}") variants = getattr(bench, "VARIANTS", None) if variants: @@ -70,10 +75,7 @@ def cmd_run(args) -> None: result = asyncio.run( run_benchmark( - adapter=adapter, - client=client, - caps=caps, - model_id=target.model_id, + routes=live, samples=samples, build_request=lambda s: bench.build_request(s, mode), parse=bench.parse, @@ -84,10 +86,18 @@ def cmd_run(args) -> None: ) ) + responses = store.load_responses() + hosts: dict = {} + for r in responses: + h = r.get("host") + if h: + hosts[h] = hosts.get(h, 0) + 1 + score_kwargs = {} if "target" in inspect.signature(bench.score).parameters: score_kwargs["target"] = target - metrics = bench.score(store.load_responses(), samples, **score_kwargs) + caps = live[0].caps # primary (first available) route's resolved caps + metrics = bench.score(responses, samples, **score_kwargs) metrics.update( { "benchmark": result_name, @@ -100,6 +110,7 @@ def cmd_run(args) -> None: "style": caps.reasoning.style.value, "true_off": caps.reasoning.true_off, }, + "hosts": hosts, # which provider(s) actually served the rows "capability_hints": result.hints, } ) @@ -108,6 +119,7 @@ def cmd_run(args) -> None: { "provider": target.provider, "model_id": target.model_id, + "routes": [rt.provider for rt in live], # providers available this run "reasoning_mode": mode, "n_completed": result.n_completed, "n_failed": result.n_failed, diff --git a/bench_core/config.py b/bench_core/config.py index c45c420..9ce7f5f 100644 --- a/bench_core/config.py +++ b/bench_core/config.py @@ -14,6 +14,7 @@ # --- provider registry ---------------------------------------------------- + @dataclass class ProviderSpec: adapter_cls: type @@ -129,6 +130,7 @@ class ProviderSpec: # targets + @dataclass class Target: name: str @@ -137,6 +139,9 @@ class Target: capabilities: dict = field(default_factory=dict) overrides: dict = field(default_factory=dict) harness_specific: dict = field(default_factory=dict) + fallbacks: list = field( + default_factory=list + ) # ordered alternate {provider, model_id, capabilities?} raw: dict = field(default_factory=dict) @@ -159,6 +164,15 @@ def load_all_targets(path: str | Path = DEFAULT_TARGETS_FILE) -> dict[str, Targe ) if not spec.get("model_id"): raise ValueError(f"target {name!r}: missing model_id") + fallbacks = spec.get("fallbacks") or [] + for fb in fallbacks: + if fb.get("provider") not in PROVIDERS: + raise ValueError( + f"target {name!r}: fallback provider {fb.get('provider')!r} " + f"unknown; known: {sorted(PROVIDERS)}" + ) + if not fb.get("model_id"): + raise ValueError(f"target {name!r}: fallback missing model_id") out[name] = Target( name=name, provider=provider, @@ -166,6 +180,7 @@ def load_all_targets(path: str | Path = DEFAULT_TARGETS_FILE) -> dict[str, Targe capabilities=spec.get("capabilities") or {}, overrides=spec.get("overrides") or {}, harness_specific=spec.get("harness_specific") or {}, + fallbacks=fallbacks, raw=spec, ) return out @@ -203,3 +218,35 @@ def build_adapter(target: Target): key_spec=list(spec.key_spec), capability_defaults=spec.capability_defaults, ) + + +def build_routes( + target: Target, benchmark: str | None = None, cli_overrides: dict | None = None +): + """Ordered provider routes: the primary, then each declared fallback. Each + route resolves its OWN provider's adapter + capabilities. A proprietary + target with no fallbacks yields a single route (so it stops on failure).""" + from bench_core.runner import Route + + specs = [target] + [ + Target( + name=target.name, + provider=fb["provider"], + model_id=fb["model_id"], + capabilities=fb.get("capabilities") or {}, + overrides=target.overrides, + raw=fb, + ) + for fb in target.fallbacks + ] + return [ + Route( + provider=t.provider, + model_id=t.model_id, + adapter=build_adapter(t), + caps=resolve_capabilities( + t, benchmark=benchmark, cli_overrides=cli_overrides + ), + ) + for t in specs + ] diff --git a/bench_core/runner.py b/bench_core/runner.py index 90b874f..9673a06 100644 --- a/bench_core/runner.py +++ b/bench_core/runner.py @@ -1,4 +1,5 @@ -"""The one execution harness: rate-limit, semaphore, retry, resume, checkpoint.""" +"""The one execution harness: rate-limit, worker pool, provider failover, retry, +resume, checkpoint.""" from __future__ import annotations @@ -10,6 +11,19 @@ from bench_core.results import RunStore _RETRYABLE = {ErrorKind.RATE_LIMITED, ErrorKind.TRANSIENT, ErrorKind.EMPTY_CONTENT} +# Advance to the next provider route only when the host itself won't serve. +# NOT on EMPTY_CONTENT: the backup runs the same weights and returns the same +# empty output, so failing over there just doubles the bill for a dead sample. +_FAILOVER = {ErrorKind.FATAL, ErrorKind.RATE_LIMITED, ErrorKind.TRANSIENT} + + +@dataclass +class Route: + provider: str + model_id: str + adapter: object + caps: object + client: object = None @dataclass @@ -48,10 +62,7 @@ async def acquire(self): async def run_benchmark( *, - adapter, - client, - caps, - model_id: str, + routes: list[Route], samples, build_request, parse, @@ -77,24 +88,44 @@ def done(r): async def process(sample): req = build_request(sample) - for attempt in range(max_retries + 1): - await limiter.acquire() - try: - resp, hints = await asyncio.to_thread( - execute, adapter, client, req, caps, model_id + last_err = None + for route in routes: + outcome = None + for attempt in range(max_retries + 1): + await limiter.acquire() + try: + resp, hints = await asyncio.to_thread( + execute, + route.adapter, + route.client, + req, + route.caps, + route.model_id, + ) + except BenchError as be: + last_err = be + kind = be.classification.kind + if kind in _RETRYABLE and attempt < max_retries: + if backoff_base > 0: + await asyncio.sleep(backoff_base * (2**attempt)) + continue + outcome = kind + break + await _record_success( + sample, + resp, + hints, + parse, + store, + result, + lock, + id_key, + route.provider, ) - except BenchError as be: - kind = be.classification.kind - if kind in _RETRYABLE and attempt < max_retries: - if backoff_base > 0: - await asyncio.sleep(backoff_base * (2**attempt)) - continue - await _record_failure(sample, be, store, result, lock, id_key) return - await _record_success( - sample, resp, hints, parse, store, result, lock, id_key - ) - return + if outcome not in _FAILOVER: + break # a genuine non-answer from this host; a backup won't help + await _record_failure(sample, last_err, store, result, lock, id_key) # Bounded worker pool: build_request runs only when a worker pulls a sample, # so at most max_in_flight requests (and their decoded images) exist at once. @@ -116,13 +147,16 @@ async def worker(): return result -async def _record_success(sample, resp, hints, parse, store, result, lock, id_key): +async def _record_success( + sample, resp, hints, parse, store, result, lock, id_key, host +): prediction = parse(resp, sample) record = { id_key: sample[id_key], "response": resp.text, "prediction": prediction, "reasoning_tokens": resp.reasoning_tokens, + "host": host, # which provider actually served this row (failover audit) } if hints: record["capability_hints"] = hints diff --git a/bench_core/targets.yaml b/bench_core/targets.yaml index b2dc8aa..6cdc029 100644 --- a/bench_core/targets.yaml +++ b/bench_core/targets.yaml @@ -15,6 +15,11 @@ targets: model_id: thinkingmachines/inkling-small capabilities: reasoning: { extra: { toggle: effort }, off_value: none, on_value: high, true_off: true } + # if OpenRouter can't serve, fall back to the same weights on Fireworks + # (host is recorded per row; proprietary targets declare no fallbacks -> stop) + fallbacks: + - provider: fireworks + model_id: accounts/fireworks/models/inkling-small grok-4.3: provider: openrouter diff --git a/tests/test_config.py b/tests/test_config.py index 916e2e4..27a63a0 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -6,9 +6,12 @@ import textwrap +import pytest + from bench_core.capabilities import AudioShape, ReasoningStyle from bench_core.config import ( build_adapter, + build_routes, load_all_targets, load_target, resolve_capabilities, @@ -158,3 +161,53 @@ def test_build_adapter_maps_provider_to_class(tmp_path): assert "fireworks_api_key" in [k.lower() for k in a.key_spec] assert a.base_url and "fireworks" in a.base_url assert isinstance(build_adapter(load_target("b", path)), GeminiAdapter) + + +def test_fallbacks_build_ordered_routes_with_own_caps(tmp_path): + path = _targets( + tmp_path, + """ + inkling-small: + provider: openrouter + model_id: thinkingmachines/inkling-small + fallbacks: + - provider: fireworks + model_id: accounts/fireworks/models/inkling-small + """, + ) + routes = build_routes(load_target("inkling-small", path)) + assert [(r.provider, r.model_id) for r in routes] == [ + ("openrouter", "thinkingmachines/inkling-small"), + ("fireworks", "accounts/fireworks/models/inkling-small"), + ] + # each route resolves its OWN provider's caps, not the primary's + assert routes[0].caps.reasoning.style is ReasoningStyle.REASONING_BODY + assert routes[1].caps.reasoning.style is ReasoningStyle.EFFORT + + +def test_no_fallbacks_is_single_route(tmp_path): + path = _targets( + tmp_path, + """ + solo: + provider: interfaze + model_id: interfaze-x + """, + ) + assert len(build_routes(load_target("solo", path))) == 1 + + +def test_fallback_is_validated(tmp_path): + path = _targets( + tmp_path, + """ + bad: + provider: openrouter + model_id: or/x + fallbacks: + - provider: nope + model_id: y + """, + ) + with pytest.raises(ValueError): + load_all_targets(path) diff --git a/tests/test_runner.py b/tests/test_runner.py index 74c05c7..4e05273 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -8,7 +8,13 @@ from bench_core.request import Message, ReasoningSpec, Request, TextPart from bench_core.response import Response from bench_core.results import RunStore -from bench_core.runner import run_benchmark +from bench_core.runner import Route, run_benchmark + + +def _route(adapter, provider="test", caps=None): + return Route( + provider=provider, model_id="m", adapter=adapter, caps=caps, client=None + ) class FakeAPIError(Exception): @@ -60,10 +66,7 @@ def _parse(resp, sample): async def _run(adapter, samples, store, **kw): return await run_benchmark( - adapter=adapter, - client=None, - caps=CAPS, - model_id="m", + routes=[_route(adapter, caps=CAPS)], samples=samples, build_request=_build_request, parse=_parse, @@ -136,10 +139,7 @@ def parse(resp, sample): return resp.text await run_benchmark( - adapter=adapter, - client=None, - caps=CAPS, - model_id="m", + routes=[_route(adapter, caps=CAPS)], samples=[{"id": i} for i in ids], build_request=build_request, parse=parse, @@ -170,10 +170,7 @@ async def test_capability_hints_are_aggregated(tmp_path): } ) result = await run_benchmark( - adapter=adapter, - client=None, - caps=caps, - model_id="m", + routes=[_route(adapter, caps=caps)], samples=[{"id": "a"}], build_request=_build_request, parse=_parse, @@ -181,3 +178,72 @@ async def test_capability_hints_are_aggregated(tmp_path): backoff_base=0.0, ) assert any("floor" in h.lower() for h in result.hints) + + +async def test_failover_to_next_route_on_fatal(tmp_path): + dead = MapAdapter({"a": [FakeAPIError(404, "no such model")]}) # FATAL + live = MapAdapter({"a": ["ok"]}) + store = RunStore("t", "m", root=tmp_path) + routes = [_route(dead, "openrouter", CAPS), _route(live, "fireworks", CAPS)] + result = await run_benchmark( + routes=routes, + samples=[{"id": "a"}], + build_request=_build_request, + parse=_parse, + store=store, + backoff_base=0.0, + ) + assert result.n_completed == 1 + rec = store.load_responses()[0] + assert rec["host"] == "fireworks" # served by the fallback, recorded + assert live.calls["a"] == 1 + + +async def test_no_failover_on_empty_content(tmp_path): + # EMPTY_CONTENT on every retry: the backup runs the same weights, so failing + # over would just double the bill — it must NOT be tried. + empty = MapAdapter({"a": ["", "", "", ""]}) + backup = MapAdapter({"a": ["ok"]}) + store = RunStore("t", "m", root=tmp_path) + routes = [_route(empty, "openrouter", CAPS), _route(backup, "fireworks", CAPS)] + result = await run_benchmark( + routes=routes, + samples=[{"id": "a"}], + build_request=_build_request, + parse=_parse, + store=store, + backoff_base=0.0, + max_retries=3, + ) + assert result.n_failed == 1 + assert "a" not in backup.calls # backup never called + + +async def test_all_routes_fail_records_failure(tmp_path): + d1 = MapAdapter({"a": [FakeAPIError(404, "x")]}) + d2 = MapAdapter({"a": [FakeAPIError(404, "y")]}) + store = RunStore("t", "m", root=tmp_path) + result = await run_benchmark( + routes=[_route(d1, "openrouter", CAPS), _route(d2, "fireworks", CAPS)], + samples=[{"id": "a"}], + build_request=_build_request, + parse=_parse, + store=store, + backoff_base=0.0, + ) + assert result.n_failed == 1 + assert d2.calls["a"] == 1 # the last route was tried + + +async def test_single_route_records_host(tmp_path): + live = MapAdapter({"a": ["ok"]}) + store = RunStore("t", "m", root=tmp_path) + await run_benchmark( + routes=[_route(live, "interfaze", CAPS)], + samples=[{"id": "a"}], + build_request=_build_request, + parse=_parse, + store=store, + backoff_base=0.0, + ) + assert store.load_responses()[0]["host"] == "interfaze" From 377406de588ef0b773df145811defc4a85bc6ea8 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Mon, 17 Aug 2026 09:31:25 +0530 Subject: [PATCH 34/38] chore: fix all ruff + ty errors repo-wide bench_core + migrated benchmarks + scripts are ruff- and ty-clean. Key fixes: reasoning.build_reasoning uses assert_never (exhaustive guard, no unreachable raise, no |None cascade); Route.caps typed Capabilities; lazy _DATASET globals and mixed part/row lists annotated; reducto client boundary-typed. Vendored scorers/harnesses excluded from ruff (extend-exclude). Folds in concurrent working-tree edits. --- .github/workflows/add-target.yml | 2 +- .github/workflows/benchmark.yml | 3 +- .github/workflows/tests.yml | 1 - README.md | 7 +- bench_core/__main__.py | 8 +- bench_core/capabilities.py | 49 +-- bench_core/cli.py | 8 +- bench_core/config.py | 6 - bench_core/errors.py | 4 +- bench_core/execute.py | 4 +- bench_core/media.py | 12 +- bench_core/providers/anthropic.py | 6 +- bench_core/providers/gemini.py | 4 +- bench_core/reasoning.py | 8 +- bench_core/request.py | 2 +- bench_core/response.py | 2 +- bench_core/results.py | 10 - bench_core/runner.py | 3 +- benchmarks/asr/bench.py | 5 +- benchmarks/gpqa/bench.py | 2 +- benchmarks/mmmu_pro/bench.py | 5 +- benchmarks/obj_detection/bench.py | 3 +- benchmarks/ocrbench_v2/bench.py | 3 +- benchmarks/ocrbench_v2/eval_scripts/eval.py | 28 +- .../eval_scripts/eval_text_spotting_en.py | 8 +- benchmarks/olmocr/bench/benchmark.py | 251 ++++++++++--- benchmarks/olmocr/bench/katex/render.py | 49 ++- benchmarks/olmocr/bench/report.py | 26 +- .../olmocr/bench/runners/run_reducto.py | 1 + benchmarks/olmocr/bench/tests.py | 348 +++++++++++++++--- benchmarks/olmocr/bench/utils.py | 29 +- benchmarks/olmocr/data/renderpdf.py | 32 +- benchmarks/olmocr/olmocr_bench_reducto.py | 4 +- benchmarks/olmocr/repeatdetect.py | 2 +- benchmarks/spider2_lite/fetch_data.py | 10 +- pyproject.toml | 13 +- scripts/plot_inkling_radar.py | 1 - scripts/sheet_columns.py | 184 ++++++--- src/commons_reducto.py | 7 +- tests/test_encode_gemini.py | 7 +- tests/test_reasoning.py | 10 +- uv.lock | 31 ++ 42 files changed, 887 insertions(+), 311 deletions(-) diff --git a/.github/workflows/add-target.yml b/.github/workflows/add-target.yml index fe26e74..1ead001 100644 --- a/.github/workflows/add-target.yml +++ b/.github/workflows/add-target.yml @@ -37,7 +37,7 @@ jobs: enable-cache: true - run: uv sync - name: Validate + append target - env: # via env, not inline ${{ }}, so JSON quotes can't break/inject the shell + env: T_NAME: ${{ github.event.inputs.name }} T_PROVIDER: ${{ github.event.inputs.provider }} T_MODEL: ${{ github.event.inputs.model_id }} diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a0f4319..ebc160f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -16,7 +16,7 @@ on: description: "reasoning override: off|low|medium|high (blank = benchmark default)" default: "" schedule: - - cron: "0 6 * * 1" # weekly (Mon) plumbing health check — only ci_regression targets + - cron: "0 6 * * 1" push: branches: [main] paths: [bench_core/targets.yaml] # a merged target change runs a full benchmark for it @@ -104,7 +104,6 @@ jobs: report: needs: [plan, run] - # only when something actually ran; skip the runner entirely on a no-op push if: always() && needs.plan.outputs.has_work == 'true' runs-on: ubuntu-latest steps: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index be7e71d..04f0144 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,7 +5,6 @@ on: branches: [main] # post-merge sanity; feature branches are covered by their PR pull_request: -# supersede stale runs on the same ref instead of piling them up concurrency: group: tests-${{ github.ref }} cancel-in-progress: true diff --git a/README.md b/README.md index 9d4302a..9ed06f0 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,7 @@ uv run python -m bench_core run --target gpt-5.5 --benchmark ocrbench_v2 --sampl uv run python scripts/report_scores.py # view scores ``` -`--sample N` runs the first N samples — a *smoke*. It **streams** only those N -rows, so smoking a heavy image benchmark (e.g. `ocrbench_v2`, ~10k images) no -longer downloads the whole split first. Smokes write to `results/_smoke/…`, -isolated from full runs and hidden from `report_scores.py`, so they can't clobber -a real score. Omit `--sample` for the full benchmark. Re-running resumes from the -checkpoint. Add `--reasoning off|low|medium|high` to override the target's default. +`--sample N` runs the first N samples — a *smoke*. It **streams** only those N rows, so smoking a heavy image benchmark (e.g. `ocrbench_v2`, ~10k images) no longer downloads the whole split first. Smokes write to `results/_smoke/…`, isolated from full runs and hidden from `report_scores.py`, so they can't clobber a real score. Omit `--sample` for the full benchmark. Re-running resumes from the checkpoint. Add `--reasoning off|low|medium|high` to override the target's default. > A **full** image-benchmark run loads the whole split into RAM (RefCOCO `val` ≈ > 8.8k images → several GB). Smoke with `--sample` first; if a full local run runs diff --git a/bench_core/__main__.py b/bench_core/__main__.py index 722410f..bd0698a 100644 --- a/bench_core/__main__.py +++ b/bench_core/__main__.py @@ -1,14 +1,12 @@ +# Some provider SDKs (google-genai) leave non-daemon background threads that keep +# the process alive after the run finishes and every result is already fsync'd. + import os import sys from bench_core.cli import main main() -# Some provider SDKs (google-genai) leave non-daemon background threads that keep -# the process alive after the run finishes and every result is already fsync'd. -# This is a one-shot CLI with nothing left to do, so exit hard rather than hang a -# CI job until its timeout. A SystemExit from an error propagates before this and -# keeps its non-zero code. sys.stdout.flush() sys.stderr.flush() os._exit(0) diff --git a/bench_core/capabilities.py b/bench_core/capabilities.py index 17d20e5..906e864 100644 --- a/bench_core/capabilities.py +++ b/bench_core/capabilities.py @@ -10,47 +10,24 @@ class ReasoningStyle(str, Enum): - """How a host expresses "think this much". - - The vocabulary is genuinely different per host — this enum names the - mechanism so the adapter can translate an abstract mode into the concrete - param without a benchmark-level `if provider == ...`. - """ - - EFFORT = ( - "effort" # OpenAI/Fireworks/Interfaze: reasoning_effort="none|low|high|..." - ) - THINKING_LEVEL = "thinking_level" # Gemini 3.x: thinking_config.thinking_level enum - THINKING_BUDGET = ( - "thinking_budget" # Gemini 2.5: thinking_config.thinking_budget int - ) - DISABLED_BLOCK = ( - "disabled_block" # Anthropic: thinking={"type":"disabled"|"enabled"} - ) - REASONING_BODY = ( - "reasoning_body" # OpenRouter: extra_body.reasoning{enabled|effort} - ) + EFFORT = "effort" # OpenAI/Fireworks/Interfaze: reasoning_effort + THINKING_LEVEL = "thinking_level" # Gemini 3.x: thinking_config.thinking_level + THINKING_BUDGET = "thinking_budget" # Gemini 2.5: thinking_budget int + DISABLED_BLOCK = "disabled_block" # Anthropic: thinking type disabled/enabled + REASONING_BODY = "reasoning_body" # OpenRouter: extra_body.reasoning NONE = "none" # non-reasoning model: send nothing class AudioShape(str, Enum): - FILE_BLOCK = ( - "file_block" # Interfaze: {"type":"file","file":{"file_data": data-uri}} - ) - AUDIO_URL = ( - "audio_url" # Fireworks: {"type":"audio_url","audio_url":{"url": data-uri}} - ) - INPUT_AUDIO = ( - "input_audio" # OpenRouter: {"type":"input_audio","input_audio":{data,format}} - ) - GEMINI_PART = "gemini_part" # Gemini: types.Part.from_bytes(mime_type="audio/...") + FILE_BLOCK = "file_block" # Interfaze: file block, data-uri + AUDIO_URL = "audio_url" # Fireworks: audio_url data-uri + INPUT_AUDIO = "input_audio" # OpenRouter: input_audio {data, format} + GEMINI_PART = "gemini_part" # Gemini: Part.from_bytes(mime_type="audio/...") NONE = "none" class ImageShape(str, Enum): - IMAGE_URL = ( - "image_url" # OpenAI-family: {"type":"image_url","image_url":{"url": data-uri}} - ) + IMAGE_URL = "image_url" # OpenAI-family: image_url data-uri ANTHROPIC_SOURCE = "anthropic_source" # {"type":"image","source":{base64}} GEMINI_PART = "gemini_part" # types.Part.from_bytes(mime_type="image/...") NONE = "none" @@ -63,12 +40,6 @@ class ResponseShape(str, Enum): def deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> dict: - """Recursively merge `override` onto `base`, returning a new dict. - - Nested dicts merge key-by-key; every other value (scalar, list) is replaced - wholesale by `override`. Neither input is mutated — the resolution chain - (adapter defaults ← target ← CLI) reuses the same base dict repeatedly. - """ result = copy.deepcopy(dict(base)) for key, val in override.items(): existing = result.get(key) diff --git a/bench_core/cli.py b/bench_core/cli.py index 6103390..e8c5198 100644 --- a/bench_core/cli.py +++ b/bench_core/cli.py @@ -13,7 +13,6 @@ from bench_core.results import RunStore from bench_core.runner import run_benchmark -# benchmark short-name -> module exposing NAME/load_samples/build_request/parse/score BENCHMARKS = { "gpqa": "benchmarks.gpqa.bench", "asr": "benchmarks.asr.bench", @@ -66,10 +65,8 @@ def cmd_run(args) -> None: result_name = bench.NAME print(f"Loading {result_name} samples...") - # sampled runs are smokes: isolate them under results/_smoke/ so they can - # never clobber or shadow a full run's metrics (results/ is gitignored, so a - # clobbered full-run metrics.json would be unrecoverable). - root = "results" if args.sample is None else "results/_smoke" + + root = "results" if args.sample is None else "results/_smoke" # sampled runs are smokes, so isolated store = RunStore(result_name, target.name, root=root) d = bench.DEFAULTS @@ -147,7 +144,6 @@ def cmd_list_targets(args) -> None: def main(argv=None) -> None: - # pick up provider keys from a repo-root .env (adapters read os.getenv) try: from dotenv import load_dotenv diff --git a/bench_core/config.py b/bench_core/config.py index 9ce7f5f..e928041 100644 --- a/bench_core/config.py +++ b/bench_core/config.py @@ -12,8 +12,6 @@ from bench_core.providers.gemini import GeminiAdapter from bench_core.providers.openai_compat import OpenAICompatAdapter -# --- provider registry ---------------------------------------------------- - @dataclass class ProviderSpec: @@ -130,7 +128,6 @@ class ProviderSpec: # targets - @dataclass class Target: name: str @@ -223,9 +220,6 @@ def build_adapter(target: Target): def build_routes( target: Target, benchmark: str | None = None, cli_overrides: dict | None = None ): - """Ordered provider routes: the primary, then each declared fallback. Each - route resolves its OWN provider's adapter + capabilities. A proprietary - target with no fallbacks yields a single route (so it stops on failure).""" from bench_core.runner import Route specs = [target] + [ diff --git a/bench_core/errors.py b/bench_core/errors.py index f3c41ed..066961f 100644 --- a/bench_core/errors.py +++ b/bench_core/errors.py @@ -6,9 +6,7 @@ class ErrorKind(str, Enum): - PARAM_REJECTED = ( - "param_rejected" - ) + PARAM_REJECTED = "param_rejected" RATE_LIMITED = "rate_limited" TRANSIENT = "transient" # timeout / 5xx / connection -> backoff + retry EMPTY_CONTENT = "empty_content" diff --git a/bench_core/execute.py b/bench_core/execute.py index ac64a85..de58ac5 100644 --- a/bench_core/execute.py +++ b/bench_core/execute.py @@ -1,4 +1,4 @@ -"""The fallback ladder: one request through an adapter, self-healing declared-cap gaps.""" +# The fallback ladder: one request through an adapter, self-healing declared-cap gaps. from __future__ import annotations @@ -50,8 +50,6 @@ def _force_reasoning_floor(caps: Capabilities) -> tuple[Capabilities, str]: def _apply_action( c: Classification, req: Request, caps: Capabilities ) -> tuple[Request, Capabilities, str] | None: - """Return a mutated (req, caps, hint), or None if the action can't be - applied here (escalates to fatal).""" if c.action == "raise_thinking_floor": caps, hint = _raise_thinking_floor(caps) return req, caps, hint diff --git a/bench_core/media.py b/bench_core/media.py index f004763..f5af87d 100644 --- a/bench_core/media.py +++ b/bench_core/media.py @@ -1,4 +1,4 @@ -"""Media block builders — one per host shape (image + audio).""" +# Media block builders — one per host shape (image + audio). from __future__ import annotations @@ -11,12 +11,9 @@ from bench_core.request import AudioPart, ImagePart +#Marker for a native `google.genai` part @dataclass(frozen=True) class GeminiPart: - """Marker for a native `google.genai` part. Kept SDK-free here so media - logic is testable without the SDK; the Gemini adapter maps it to - `types.Part.from_bytes(data=..., mime_type=...)`.""" - data: bytes mime: str @@ -83,9 +80,6 @@ def image_block(part: ImagePart, shape: ImageShape) -> Any: def encode_image( pil_image, mime: str = "image/jpeg", max_side: int | None = None ) -> ImagePart: - """PIL image -> ImagePart, doing the conversions every consumer needs once: - RGB-convert for JPEG (JPEG can't hold alpha; the omission crashed 8/9 OCR - variants on RGBA/P images), optional longest-side downscale, then encode.""" from PIL import Image img = pil_image @@ -94,7 +88,7 @@ def encode_image( longest = max(w, h) if longest > max_side: scale = max_side / longest - img = img.resize((round(w * scale), round(h * scale)), Image.LANCZOS) + img = img.resize((round(w * scale), round(h * scale)), Image.Resampling.LANCZOS) fmt = _subtype(mime).upper() if fmt in ("JPG", "JPEG"): diff --git a/bench_core/providers/anthropic.py b/bench_core/providers/anthropic.py index 012a530..b1573d1 100644 --- a/bench_core/providers/anthropic.py +++ b/bench_core/providers/anthropic.py @@ -61,7 +61,11 @@ def encode(self, req: Request, caps: Capabilities, model_id: str) -> dict: thinking_on = (inj.anthropic_thinking or {}).get("type") == "enabled" if thinking_on: - budget = inj.anthropic_thinking.get("budget_tokens", 0) if inj.anthropic_thinking else 0 + budget = ( + inj.anthropic_thinking.get("budget_tokens", 0) + if inj.anthropic_thinking + else 0 + ) max_tokens = req.max_tokens or extra.get("on_max_tokens", _DEFAULT_ON_MAX) if max_tokens <= budget: max_tokens = budget + _BUDGET_MARGIN diff --git a/bench_core/providers/gemini.py b/bench_core/providers/gemini.py index 81ef44c..50b74e6 100644 --- a/bench_core/providers/gemini.py +++ b/bench_core/providers/gemini.py @@ -48,7 +48,9 @@ def encode(self, req: Request, caps: Capabilities, model_id: str) -> dict: inj = build_reasoning(req.reasoning.mode, caps.reasoning) thinking = None if inj.thinking_level is not None: - thinking = types.ThinkingConfig(thinking_level=types.ThinkingLevel(inj.thinking_level)) + thinking = types.ThinkingConfig( + thinking_level=types.ThinkingLevel(inj.thinking_level) + ) elif inj.thinking_budget is not None: thinking = types.ThinkingConfig(thinking_budget=inj.thinking_budget) diff --git a/bench_core/reasoning.py b/bench_core/reasoning.py index 57ef362..720e09d 100644 --- a/bench_core/reasoning.py +++ b/bench_core/reasoning.py @@ -1,9 +1,9 @@ -"""Translate an abstract reasoning mode into the host's concrete directive.""" +# Translate an abstract reasoning mode into the host's concrete directive. from __future__ import annotations from dataclasses import dataclass, field -from typing import Any +from typing import Any, assert_never from bench_core.capabilities import ReasoningCap, ReasoningStyle @@ -89,4 +89,6 @@ def build_reasoning(mode: str, cap: ReasoningCap) -> ReasoningInjection: extra_body["provider"] = {"only": list(pin)} return ReasoningInjection(extra_body=extra_body, allow_temperature=allow_temp) - raise ValueError(f"unhandled reasoning style {style!r}") + assert_never( + style + ) # exhaustive over ReasoningStyle; ty errors if a member is added diff --git a/bench_core/request.py b/bench_core/request.py index 94d861a..08df66d 100644 --- a/bench_core/request.py +++ b/bench_core/request.py @@ -1,4 +1,4 @@ -"""Provider-agnostic request the benchmark layer builds.""" +# Provider-agnostic request the benchmark layer builds. from __future__ import annotations diff --git a/bench_core/response.py b/bench_core/response.py index 43dfa96..d2ccd67 100644 --- a/bench_core/response.py +++ b/bench_core/response.py @@ -1,4 +1,4 @@ -"""Normalized response — one shape regardless of host.""" +# Normalized response — one shape regardless of host. from __future__ import annotations diff --git a/bench_core/results.py b/bench_core/results.py index 6788bb2..4dd7a48 100644 --- a/bench_core/results.py +++ b/bench_core/results.py @@ -1,5 +1,3 @@ -"""The single result contract: one slug, one layout, one content-based discovery.""" - from __future__ import annotations import json @@ -12,13 +10,7 @@ def model_slug(name: str) -> str: - """The ONE slug rule. Strip any provider path, then collapse to - lowercase-dash — so fireworks/openrouter/google spellings of the same model - resolve to the same directory.""" leaf = name.rsplit("/", 1)[-1] - # Preserve dots/underscores/dashes (filesystem-safe and keeps version numbers - # like "3.7" intact) so the slug matches the target's YAML `name` and no - # un-mangling step is ever needed. Only collapse genuinely unsafe chars. return re.sub(r"[^a-z0-9._-]+", "-", leaf.lower()).strip("-") @@ -76,8 +68,6 @@ def write_run(self, run: dict) -> None: def discover(root: Path | str = DEFAULT_ROOT) -> list[dict]: - """Every metrics.json under the layout — content-based, the single source - the report reads.""" root = Path(root) out = [] for path in sorted(root.glob("*/*/metrics.json")): diff --git a/bench_core/runner.py b/bench_core/runner.py index 9673a06..cb80146 100644 --- a/bench_core/runner.py +++ b/bench_core/runner.py @@ -6,6 +6,7 @@ import asyncio from dataclasses import dataclass, field +from bench_core.capabilities import Capabilities from bench_core.errors import ErrorKind from bench_core.execute import BenchError, execute from bench_core.results import RunStore @@ -22,7 +23,7 @@ class Route: provider: str model_id: str adapter: object - caps: object + caps: Capabilities client: object = None diff --git a/benchmarks/asr/bench.py b/benchmarks/asr/bench.py index 3780e8e..3510f10 100644 --- a/benchmarks/asr/bench.py +++ b/benchmarks/asr/bench.py @@ -1,9 +1,10 @@ -"""VoxPopuli-Cleaned-AA ASR: transcribe audio, score corpus + time-weighted WER.""" +# VoxPopuli-Cleaned-AA ASR: transcribe audio, score corpus + time-weighted WER. from __future__ import annotations import re import unicodedata +from typing import Any from jiwer import cer, wer @@ -92,7 +93,7 @@ def _sample_metric(fn, gt_norm: str, hyp_norm: str) -> float: def score(records: list[dict], samples: list[dict]) -> dict: by_id = {s["id"]: s for s in samples} - rows = [] + rows: list[dict[str, Any]] = [] for r in records: s = by_id.get(r["id"]) if s is None: diff --git a/benchmarks/gpqa/bench.py b/benchmarks/gpqa/bench.py index b9ed208..3c2b1f1 100644 --- a/benchmarks/gpqa/bench.py +++ b/benchmarks/gpqa/bench.py @@ -1,4 +1,4 @@ -"""GPQA Diamond: 198 expert MCQs; deterministic choice shuffle; letter-match accuracy.""" +# GPQA Diamond: 198 expert MCQs; deterministic choice shuffle; letter-match accuracy. from __future__ import annotations diff --git a/benchmarks/mmmu_pro/bench.py b/benchmarks/mmmu_pro/bench.py index 4244314..0f36d8d 100644 --- a/benchmarks/mmmu_pro/bench.py +++ b/benchmarks/mmmu_pro/bench.py @@ -5,6 +5,7 @@ import ast import re from collections import defaultdict +from typing import Any from bench_core.media import encode_image from bench_core.request import Message, ReasoningSpec, Request, TextPart @@ -19,7 +20,7 @@ _SPLIT = "test" _CONFIGS = {"standard": "standard (10 options)", "vision": "vision"} _MAX_IMAGE_SIDE = 1536 -_DATASET = None # full-run split; images read lazily by idx to bound memory +_DATASET: Any = None # full-run split; images read lazily by idx to bound memory _LETTERS = list("ABCDEFGHIJ") _PROMPT_STANDARD = ( @@ -131,7 +132,7 @@ def build_request(sample: dict, mode: str) -> Request: prompt = _PROMPT_STANDARD.format( question=sample["question"], options=_options_block(sample["options"]) ) - parts = [TextPart(prompt)] + parts: list = [TextPart(prompt)] parts += [ encode_image(im, "image/jpeg", max_side=_MAX_IMAGE_SIDE) for im in pil_images ] diff --git a/benchmarks/obj_detection/bench.py b/benchmarks/obj_detection/bench.py index e9b78a3..a7e31ba 100644 --- a/benchmarks/obj_detection/bench.py +++ b/benchmarks/obj_detection/bench.py @@ -9,6 +9,7 @@ import re from collections import defaultdict +from typing import Any from bench_core.media import encode_image from bench_core.request import Message, ReasoningSpec, Request, TextPart @@ -37,7 +38,7 @@ "g": "lmms-lab/RefCOCOg", } _MAX_SIDE = 1024 -_DATASET = None # full-run split; images read lazily by idx to bound memory +_DATASET: Any = None # full-run split; images read lazily by idx to bound memory IOU_THRESHOLD = 0.5 _THRESHOLDS = [0.3, 0.5, 0.7, 0.75, 0.9] diff --git a/benchmarks/ocrbench_v2/bench.py b/benchmarks/ocrbench_v2/bench.py index df508c0..6bb3ebe 100644 --- a/benchmarks/ocrbench_v2/bench.py +++ b/benchmarks/ocrbench_v2/bench.py @@ -13,6 +13,7 @@ import tempfile from collections import defaultdict from pathlib import Path +from typing import Any from bench_core.media import encode_image from bench_core.request import Message, ReasoningSpec, Request, TextPart @@ -78,7 +79,7 @@ EN_CATEGORIES = list(dict.fromkeys(TYPE_TO_EN.values())) CN_CATEGORIES = list(dict.fromkeys(TYPE_TO_CN.values())) -_DATASET = None # lazily-held so images decode per request, not all 10k upfront +_DATASET: Any = None # lazily-held so images decode per request, not all 10k upfront def get_spotting_prompt(question: str) -> str: diff --git a/benchmarks/ocrbench_v2/eval_scripts/eval.py b/benchmarks/ocrbench_v2/eval_scripts/eval.py index c6fd162..4e8bc97 100644 --- a/benchmarks/ocrbench_v2/eval_scripts/eval.py +++ b/benchmarks/ocrbench_v2/eval_scripts/eval.py @@ -107,9 +107,11 @@ def process_predictions(input_path, output_path): # Infer eval method when missing eval_type = data_item.get("eval") if not eval_type: - if (len(data_item["answers"]) == 1 + if ( + len(data_item["answers"]) == 1 and len(data_item["answers"][0]) <= 2 - and data_item["answers"][0].isalpha()): + and data_item["answers"][0].isalpha() + ): eval_type = "multiple choice" if eval_type == "multiple choice": if not isinstance(data_item["answers"], list): @@ -119,9 +121,7 @@ def process_predictions(input_path, output_path): if not isinstance(data_item["predict"], str): data_item["score"] = 0 else: - predict = "".join( - c for c in data_item["predict"] if c.isalpha() - ) + predict = "".join(c for c in data_item["predict"] if c.isalpha()) if predict == data_item["answers"][0]: data_item["score"] = 1 @@ -468,7 +468,11 @@ def process_predictions(input_path, output_path): elif data_item["type"] == "text spotting en": # Parse bbox/content from answers if not present (HF dataset format) # GT format: variable-length polygons (8, 12, 16, 28+ coords) followed by text - if "bbox" not in data_item and "answers" in data_item and data_item["answers"]: + if ( + "bbox" not in data_item + and "answers" in data_item + and data_item["answers"] + ): bboxes, contents = [], [] for line in data_item["answers"][0].strip().split("\n"): parts = line.split(",") @@ -489,9 +493,13 @@ def process_predictions(input_path, output_path): x2, y2 = max(x_coords), max(y_coords) bboxes.append([x1, y1, x2, y1, x2, y2, x1, y2]) contents.append(text) - elif num_coords >= 8 and num_coords == len(parts) and num_coords % 2 == 1: + elif ( + num_coords >= 8 + and num_coords == len(parts) + and num_coords % 2 == 1 + ): # All-numeric text (e.g. "1700"): odd coord count means last is text - coords = [int(p.strip()) for p in parts[:num_coords - 1]] + coords = [int(p.strip()) for p in parts[: num_coords - 1]] text = parts[num_coords - 1].strip() x_coords = coords[0::2] y_coords = coords[1::2] @@ -510,7 +518,9 @@ def process_predictions(input_path, output_path): if not predict_bbox: data_item["score"] = 0 else: - data_item["score"] = spotting_evaluation_normalized(predict_bbox, data_item) + data_item["score"] = spotting_evaluation_normalized( + predict_bbox, data_item + ) else: raise ValueError("Unknown task type!") diff --git a/benchmarks/ocrbench_v2/eval_scripts/eval_text_spotting_en.py b/benchmarks/ocrbench_v2/eval_scripts/eval_text_spotting_en.py index e7821f2..1fbc210 100644 --- a/benchmarks/ocrbench_v2/eval_scripts/eval_text_spotting_en.py +++ b/benchmarks/ocrbench_v2/eval_scripts/eval_text_spotting_en.py @@ -47,11 +47,7 @@ def extract_bounding_boxes(predict_str): continue text_content = ( - str(item[4]) - .replace("\n", "") - .strip() - .strip('"') - .strip("'") + str(item[4]).replace("\n", "").strip().strip('"').strip("'") ) # Normalize inverted coordinates @@ -186,7 +182,7 @@ def process_predictions(input_path, output_path): # Polygons always have even number of coords (pairs of x,y) if num_coords % 2 == 1: # Odd count means last "coord" is actually the text - coords = [int(p.strip()) for p in parts[:num_coords - 1]] + coords = [int(p.strip()) for p in parts[: num_coords - 1]] text = parts[num_coords - 1].strip() x_coords = coords[0::2] y_coords = coords[1::2] diff --git a/benchmarks/olmocr/bench/benchmark.py b/benchmarks/olmocr/bench/benchmark.py index df17b03..acbb44e 100644 --- a/benchmarks/olmocr/bench/benchmark.py +++ b/benchmarks/olmocr/bench/benchmark.py @@ -31,8 +31,19 @@ def evaluate_candidate( - candidate_folder: str, all_tests: List[BasePDFTest], pdf_basenames: List[str], force: bool = False -) -> Tuple[float, int, List[str], List[str], Dict[str, List[float]], List[float], Dict[str, Dict[int, List[Tuple[BasePDFTest, bool, str]]]]]: + candidate_folder: str, + all_tests: List[BasePDFTest], + pdf_basenames: List[str], + force: bool = False, +) -> Tuple[ + float, + int, + List[str], + List[str], + Dict[str, List[float]], + List[float], + Dict[str, Dict[int, List[Tuple[BasePDFTest, bool, str]]]], +]: """ For the candidate folder (pipeline tool output), validate that it contains at least one .md file (i.e. repeated generations like _pg{page}_repeat{repeat}.md) for every PDF in the pdf folder. @@ -59,25 +70,40 @@ def evaluate_candidate( # Map each PDF to its corresponding MD repeats (e.g., doc1_pg1_repeat1.md, doc1_pg2_repeat2.md, etc.) pdf_to_md_files = {} - all_files = list(glob.glob(os.path.join(candidate_folder, "**/*.md"), recursive=True)) + all_files = list( + glob.glob(os.path.join(candidate_folder, "**/*.md"), recursive=True) + ) for pdf_name in pdf_basenames: md_base = os.path.splitext(pdf_name)[0] md_regex = re.compile(rf"^{re.escape(md_base)}_pg\d+_repeat\d+\.md$") - md_files = [f for f in all_files if md_regex.match(os.path.relpath(f, candidate_folder))] + md_files = [ + f for f in all_files if md_regex.match(os.path.relpath(f, candidate_folder)) + ] if not md_files and not force: candidate_errors.append( - f"Candidate '{candidate_name}' is missing MD repeats for {pdf_name} " f"(expected files matching {md_base}_pg{{page}}_repeat*.md)." + f"Candidate '{candidate_name}' is missing MD repeats for {pdf_name} " + f"(expected files matching {md_base}_pg{{page}}_repeat*.md)." ) else: pdf_to_md_files[pdf_name] = md_files if candidate_errors: - return (0.0, len(all_tests), candidate_errors, test_failures, test_type_breakdown, all_test_scores, test_results) + return ( + 0.0, + len(all_tests), + candidate_errors, + test_failures, + test_type_breakdown, + all_test_scores, + test_results, + ) # Define an inner function to evaluate a single test - def process_test(test: BasePDFTest) -> Tuple[float, str, str, List[str], Tuple[bool, str]]: + def process_test( + test: BasePDFTest, + ) -> Tuple[float, str, str, List[str], Tuple[bool, str]]: local_errors = [] test_failure = None pdf_name = test.pdf @@ -91,7 +117,9 @@ def process_test(test: BasePDFTest) -> Tuple[float, str, str, List[str], Tuple[b md_base = os.path.splitext(pdf_name)[0] md_files = pdf_to_md_files.get(pdf_name, []) # Filter MD files for the specific page corresponding to the test - page_md_files = [f for f in md_files if re.search(rf"_pg{test.page}_", os.path.basename(f))] + page_md_files = [ + f for f in md_files if re.search(rf"_pg{test.page}_", os.path.basename(f)) + ] if not page_md_files: local_errors.append( f"Candidate '{candidate_name}' is missing MD repeats for {pdf_name} page {test.page} " @@ -123,18 +151,28 @@ def process_test(test: BasePDFTest) -> Tuple[float, str, str, List[str], Tuple[b explanations.append(str(e)) test_avg = repeat_passes / num_repeats if num_repeats > 0 else 0.0 - final_passed = test_avg > 0.5 # Consider test passed if majority of repeats pass + final_passed = ( + test_avg > 0.5 + ) # Consider test passed if majority of repeats pass final_explanation = explanations[0] if explanations else "All repeats passed" # Store the test result for reporting - test_results[pdf_name][test.page].append((test, final_passed, final_explanation)) + test_results[pdf_name][test.page].append( + (test, final_passed, final_explanation) + ) if test_avg < 1.0: test_failure = ( f"Test {test.id} on {md_base} page {test.page} average pass ratio: {test_avg:.3f} " f"({repeat_passes}/{num_repeats} repeats passed). Ex: {explanations[0] if explanations else 'No explanation'}" ) - return (test_avg, test_failure, test.type, local_errors, (final_passed, final_explanation)) + return ( + test_avg, + test_failure, + test.type, + local_errors, + (final_passed, final_explanation), + ) total_test_score = 0.0 futures = [] @@ -142,7 +180,12 @@ def process_test(test: BasePDFTest) -> Tuple[float, str, str, List[str], Tuple[b with ThreadPoolExecutor(max_workers=min(os.cpu_count() or 1, 64)) as executor: futures = [executor.submit(process_test, test) for test in all_tests] # tqdm progress bar for this candidate's tests - for future in tqdm(as_completed(futures), total=len(futures), desc=f"Evaluating tests for {candidate_name}", unit="test"): + for future in tqdm( + as_completed(futures), + total=len(futures), + desc=f"Evaluating tests for {candidate_name}", + unit="test", + ): test_avg, test_failure, test_type, errors, _ = future.result() all_test_scores.append(test_avg) total_test_score += test_avg @@ -156,7 +199,15 @@ def process_test(test: BasePDFTest) -> Tuple[float, str, str, List[str], Tuple[b candidate_errors.extend(local_errors) overall_score = total_test_score / len(all_tests) if all_tests else 0.0 - return (overall_score, len(all_tests), candidate_errors, test_failures, test_type_breakdown, all_test_scores, test_results) + return ( + overall_score, + len(all_tests), + candidate_errors, + test_failures, + test_type_breakdown, + all_test_scores, + test_results, + ) def main(): @@ -171,8 +222,17 @@ def main(): action="store_true", help="Run benchmark even if some files are missing", ) - parser.add_argument("--candidate", type=str, default=None, help="Run test only for a single candidate") - parser.add_argument("--skip_baseline", action="store_true", help="Skip running baseline tests (ex. that check that basic content is present on each page)") + parser.add_argument( + "--candidate", + type=str, + default=None, + help="Run test only for a single candidate", + ) + parser.add_argument( + "--skip_baseline", + action="store_true", + help="Skip running baseline tests (ex. that check that basic content is present on each page)", + ) parser.add_argument( "--bootstrap_samples", type=int, @@ -186,10 +246,23 @@ def main(): help="Confidence level for interval calculation (default: 0.95 for 95% CI).", ) # New arguments - parser.add_argument("--sample", type=int, default=None, help="Randomly sample N tests to run instead of all tests.") - parser.add_argument("--test_report", type=str, default=None, help="Generate an HTML report of test results. Provide a filename (e.g., results.html).") parser.add_argument( - "--output_failed", type=str, default=None, help="Output a JSONL file containing tests that failed across all candidates. Provide a filename." + "--sample", + type=int, + default=None, + help="Randomly sample N tests to run instead of all tests.", + ) + parser.add_argument( + "--test_report", + type=str, + default=None, + help="Generate an HTML report of test results. Provide a filename (e.g., results.html).", + ) + parser.add_argument( + "--output_failed", + type=str, + default=None, + help="Output a JSONL file containing tests that failed across all candidates. Provide a filename.", ) args = parser.parse_args() @@ -202,7 +275,9 @@ def main(): print("Error: /pdfs folder must exist in your data directory.", file=sys.stderr) sys.exit(1) - all_pdf_files = list(glob.glob(os.path.join(pdf_folder, "**/*.pdf"), recursive=True)) + all_pdf_files = list( + glob.glob(os.path.join(pdf_folder, "**/*.pdf"), recursive=True) + ) if not all_pdf_files: print(f"Error: No PDF files found in {pdf_folder}", file=sys.stderr) @@ -234,13 +309,20 @@ def main(): for pdf in pdf_basenames: if not any(t.type == "baseline" for t in all_tests if t.pdf == pdf): - all_tests.append(BaselineTest(id=f"{pdf}_baseline", pdf=pdf, page=1, type="baseline")) + all_tests.append( + BaselineTest(id=f"{pdf}_baseline", pdf=pdf, page=1, type="baseline") + ) test_to_jsonl[all_tests[-1].id] = "baseline" for pdf in pdf_basenames: pdf_doc = PdfReader(os.path.join(pdf_folder, pdf)) for page in range(1, len(pdf_doc.pages) + 1): - if not any(test for test in all_tests if test.pdf == pdf and test.page == page) and not args.force: + if ( + not any( + test for test in all_tests if test.pdf == pdf and test.page == page + ) + and not args.force + ): print(f"No dataset entry found for pdf {pdf} page {page}") sys.exit(1) @@ -250,9 +332,13 @@ def main(): # Sample tests if requested if args.sample is not None and args.sample > 0: if args.sample >= len(all_tests): - print(f"Sample size {args.sample} is greater than or equal to the total number of tests ({len(all_tests)}). Using all tests.") + print( + f"Sample size {args.sample} is greater than or equal to the total number of tests ({len(all_tests)}). Using all tests." + ) else: - print(f"Randomly sampling {args.sample} tests out of {len(all_tests)} total tests.") + print( + f"Randomly sampling {args.sample} tests out of {len(all_tests)} total tests." + ) all_tests = random.sample(all_tests, args.sample) candidate_folders = [] @@ -266,7 +352,10 @@ def main(): candidate_folders.append(full_path) if not candidate_folders: - print("Error: No candidate pipeline folders found (subdirectories besides 'pdfs').", file=sys.stderr) + print( + "Error: No candidate pipeline folders found (subdirectories besides 'pdfs').", + file=sys.stderr, + ) sys.exit(1) candidate_folders.sort() @@ -278,9 +367,15 @@ def main(): for candidate in candidate_folders: candidate_name = os.path.basename(candidate) print(f"\nEvaluating candidate: {candidate_name}") - overall_score, total_tests, candidate_errors, test_failures, test_type_breakdown, all_test_scores, test_results = evaluate_candidate( - candidate, all_tests, pdf_basenames, args.force - ) + ( + overall_score, + total_tests, + candidate_errors, + test_failures, + test_type_breakdown, + all_test_scores, + test_results, + ) = evaluate_candidate(candidate, all_tests, pdf_basenames, args.force) # Always store test results for displaying jsonl file groupings test_results_by_candidate[candidate_name] = test_results @@ -321,10 +416,26 @@ def main(): # Calculate CI using the updated function with splits if jsonl_scores: - ci = calculate_bootstrap_ci(jsonl_scores, n_bootstrap=n_bootstrap, ci_level=ci_level, splits=jsonl_file_sizes) + ci = calculate_bootstrap_ci( + jsonl_scores, + n_bootstrap=n_bootstrap, + ci_level=ci_level, + splits=jsonl_file_sizes, + ) else: ci = (0.0, 0.0) - summary.append((candidate_name, overall_score, total_tests, candidate_errors, test_failures, test_type_breakdown, ci, all_test_scores)) + summary.append( + ( + candidate_name, + overall_score, + total_tests, + candidate_errors, + test_failures, + test_type_breakdown, + ci, + all_test_scores, + ) + ) print(f"\nCandidate: {candidate_name}") if candidate_errors: for err in candidate_errors: @@ -340,12 +451,27 @@ def main(): pass_rate = results["passed"] / results["total"] jsonl_pass_rates.append(pass_rate) - per_category_score = sum(jsonl_pass_rates) / len(jsonl_pass_rates) if jsonl_pass_rates else 0.0 - print(f" Average Score: {per_category_score * 100:.1f}% (95% CI: [{ci[0] * 100:.1f}%, {ci[1] * 100:.1f}%]) over {total_tests} tests.") + per_category_score = ( + sum(jsonl_pass_rates) / len(jsonl_pass_rates) + if jsonl_pass_rates + else 0.0 + ) + print( + f" Average Score: {per_category_score * 100:.1f}% (95% CI: [{ci[0] * 100:.1f}%, {ci[1] * 100:.1f}%]) over {total_tests} tests." + ) print("\n" + "=" * 60) print("Final Summary with 95% Confidence Intervals:") - for idx, (candidate_name, _, total_tests, candidate_errors, _, test_type_breakdown, ci, _) in enumerate(summary): + for idx, ( + candidate_name, + _, + total_tests, + candidate_errors, + _, + test_type_breakdown, + ci, + _, + ) in enumerate(summary): # Group results by jsonl file jsonl_results = {} for test in all_tests: @@ -362,8 +488,14 @@ def main(): if not candidate_errors and hasattr(test, "pdf") and hasattr(test, "page"): pdf_name = test.pdf page = test.page - if pdf_name in test_results_by_candidate.get(candidate_name, {}) and page in test_results_by_candidate[candidate_name].get(pdf_name, {}): - for t, passed, _ in test_results_by_candidate[candidate_name][pdf_name][page]: + if pdf_name in test_results_by_candidate.get( + candidate_name, {} + ) and page in test_results_by_candidate[candidate_name].get( + pdf_name, {} + ): + for t, passed, _ in test_results_by_candidate[candidate_name][ + pdf_name + ][page]: if t.id == test.id: test_result = passed break @@ -379,10 +511,21 @@ def main(): jsonl_pass_rates.append(pass_rate) # New overall score is average of per-JSONL pass rates - new_overall_score = sum(jsonl_pass_rates) / len(jsonl_pass_rates) if jsonl_pass_rates else 0.0 + new_overall_score = ( + sum(jsonl_pass_rates) / len(jsonl_pass_rates) if jsonl_pass_rates else 0.0 + ) # Update the overall_score in the summary list for later use (e.g., in permutation tests) - summary[idx] = (candidate_name, new_overall_score, total_tests, candidate_errors, summary[idx][4], test_type_breakdown, ci, summary[idx][7]) + summary[idx] = ( + candidate_name, + new_overall_score, + total_tests, + candidate_errors, + summary[idx][4], + test_type_breakdown, + ci, + summary[idx][7], + ) if candidate_errors: status = "FAILED (errors)" @@ -392,19 +535,25 @@ def main(): # Use the CI that was calculated with proper category-based bootstrap half_width = ((ci[1] - ci[0]) / 2) * 100 ciw_str = f"± {half_width:0.1f}%" - print(f"{candidate_name:20s} : Average Score: {status} {ciw_str} (average of per-JSONL scores)") + print( + f"{candidate_name:20s} : Average Score: {status} {ciw_str} (average of per-JSONL scores)" + ) # Sort the test types alphabetically for ttype in sorted(test_type_breakdown.keys()): scores = test_type_breakdown[ttype] avg = sum(scores) / len(scores) * 100 if scores else 0.0 - print(f" {ttype:8s}: {avg:0.1f}% average pass rate over {len(scores)} tests") + print( + f" {ttype:8s}: {avg:0.1f}% average pass rate over {len(scores)} tests" + ) print("\n Results by JSONL file:") for jsonl_file, results in sorted(jsonl_results.items()): if results["total"] > 0: pass_rate = (results["passed"] / results["total"]) * 100 - print(f" {jsonl_file:30s}: {pass_rate:0.1f}% ({results['passed']}/{results['total']} tests)") + print( + f" {jsonl_file:30s}: {pass_rate:0.1f}% ({results['passed']}/{results['total']} tests)" + ) print("") # Generate HTML report if requested @@ -415,7 +564,9 @@ def main(): if args.output_failed: # Identify tests that failed across all candidates all_failed_tests = [] - valid_candidates = [c for c in summary if not c[3]] # Skip candidates with errors + valid_candidates = [ + c for c in summary if not c[3] + ] # Skip candidates with errors for test in all_tests: # Track whether this test has any results @@ -428,8 +579,14 @@ def main(): if hasattr(test, "pdf") and hasattr(test, "page"): pdf_name = test.pdf page = test.page - if pdf_name in test_results_by_candidate.get(candidate_name, {}) and page in test_results_by_candidate[candidate_name].get(pdf_name, {}): - for t, passed, explanation in test_results_by_candidate[candidate_name][pdf_name][page]: + if pdf_name in test_results_by_candidate.get( + candidate_name, {} + ) and page in test_results_by_candidate[candidate_name].get( + pdf_name, {} + ): + for t, passed, explanation in test_results_by_candidate[ + candidate_name + ][pdf_name][page]: if t.id == test.id: has_results = True test_result = passed @@ -443,12 +600,18 @@ def main(): all_failed_tests.append(test) # If we have any failed tests, write them to the specified JSONL file - output_path = os.path.join(input_folder, args.output_failed) if not os.path.isabs(args.output_failed) else args.output_failed + output_path = ( + os.path.join(input_folder, args.output_failed) + if not os.path.isabs(args.output_failed) + else args.output_failed + ) if all_failed_tests: save_tests(all_failed_tests, output_path) - print(f"\nOutput {len(all_failed_tests)} tests that failed across all candidates to {output_path}") + print( + f"\nOutput {len(all_failed_tests)} tests that failed across all candidates to {output_path}" + ) else: print("\nNo tests failed across all candidates. No output file created.") diff --git a/benchmarks/olmocr/bench/katex/render.py b/benchmarks/olmocr/bench/katex/render.py index 109b877..b18fbdd 100644 --- a/benchmarks/olmocr/bench/katex/render.py +++ b/benchmarks/olmocr/bench/katex/render.py @@ -35,7 +35,9 @@ class EquationCache: def __init__(self, db_path: Optional[str] = None): if db_path is None: # Use the same cache directory as before - cache_dir = pathlib.Path.home() / ".cache" / "olmocr" / "bench" / "equations" + cache_dir = ( + pathlib.Path.home() / ".cache" / "olmocr" / "bench" / "equations" + ) cache_dir.mkdir(parents=True, exist_ok=True) db_path = str(cache_dir / "cache.db") self.db_path = db_path @@ -62,7 +64,10 @@ def load(self, eq_hash: str) -> Optional["RenderedEquation"]: with self.lock: conn = sqlite3.connect(self.db_path) c = conn.cursor() - c.execute("SELECT mathml, spans, error FROM equations WHERE eq_hash = ?", (eq_hash,)) + c.execute( + "SELECT mathml, spans, error FROM equations WHERE eq_hash = ?", + (eq_hash,), + ) row = c.fetchone() conn.close() if row: @@ -160,7 +165,9 @@ def get_equation_hash(equation, bg_color="white", text_color="black", font_size= # Global thread pool executor with a fixed number of threads # Each thread will maintain its own Playwright instance -_render_executor = ThreadPoolExecutor(max_workers=8, thread_name_prefix="playwright-render") +_render_executor = ThreadPoolExecutor( + max_workers=8, thread_name_prefix="playwright-render" +) def _cleanup_executor(): @@ -209,7 +216,9 @@ def _get_thread_local_browser(): return owner -def _render_in_executor(equation, bg_color, text_color, font_size, use_cache, debug_dom, eq_hash): +def _render_in_executor( + equation, bg_color, text_color, font_size, use_cache, debug_dom, eq_hash +): """ Function to be run in the executor thread pool. Each thread maintains its own Playwright instance. @@ -238,7 +247,9 @@ def _do_render(context, equation, bg_color, text_color, font_size, debug_dom): katex_js_path = os.path.join(script_dir, "katex.min.js") if not os.path.exists(katex_css_path) or not os.path.exists(katex_js_path): - raise FileNotFoundError(f"KaTeX files not found. Please ensure katex.min.css and katex.min.js are in {script_dir}") + raise FileNotFoundError( + f"KaTeX files not found. Please ensure katex.min.css and katex.min.js are in {script_dir}" + ) # Create a new page. page = context.new_page() @@ -277,7 +288,9 @@ def _do_render(context, equation, bg_color, text_color, font_size, debug_dom): katex_loaded = page.evaluate("typeof katex !== 'undefined'") if not katex_loaded: page.close() - raise RuntimeError("KaTeX library failed to load. Check your katex.min.js file.") + raise RuntimeError( + "KaTeX library failed to load. Check your katex.min.js file." + ) try: error_message = page.evaluate(f""" @@ -399,7 +412,16 @@ def render_equation( return cached # Submit the rendering task to the thread pool executor - future = _render_executor.submit(_render_in_executor, equation, bg_color, text_color, font_size, use_cache, debug_dom, eq_hash) + future = _render_executor.submit( + _render_in_executor, + equation, + bg_color, + text_color, + font_size, + use_cache, + debug_dom, + eq_hash, + ) # Wait for the result rendered_eq = future.result() @@ -411,7 +433,9 @@ def render_equation( return rendered_eq -def compare_rendered_equations(reference: RenderedEquation, hypothesis: RenderedEquation) -> bool: +def compare_rendered_equations( + reference: RenderedEquation, hypothesis: RenderedEquation +) -> bool: """ Compare two RenderedEquation objects. First, check if the normalized MathML of the hypothesis is contained within that of the reference. @@ -424,7 +448,11 @@ def extract_inner(mathml: str) -> str: soup = BeautifulSoup(mathml, "xml") semantics = soup.find("semantics") if semantics: - inner_parts = [str(child) for child in semantics.contents if getattr(child, "name", None) != "annotation"] + inner_parts = [ + str(child) + for child in semantics.contents + if getattr(child, "name", None) != "annotation" + ] return "".join(inner_parts) else: return str(soup) @@ -451,7 +479,8 @@ def expand_span_info(span_info: SpanInfo) -> list[SpanInfo]: SpanInfo( c, BoundingBox( - span_info.bounding_box.x + (span_info.bounding_box.width * index) / total_elems, + span_info.bounding_box.x + + (span_info.bounding_box.width * index) / total_elems, span_info.bounding_box.y, span_info.bounding_box.width / total_elems, span_info.bounding_box.height, diff --git a/benchmarks/olmocr/bench/report.py b/benchmarks/olmocr/bench/report.py index 939d38e..bf8bac2 100644 --- a/benchmarks/olmocr/bench/report.py +++ b/benchmarks/olmocr/bench/report.py @@ -10,7 +10,11 @@ def generate_html_report( - test_results_by_candidate: Dict[str, Dict[str, Dict[int, List[Tuple[BasePDFTest, bool, str]]]]], pdf_folder: str, output_file: str + test_results_by_candidate: Dict[ + str, Dict[str, Dict[int, List[Tuple[BasePDFTest, bool, str]]]] + ], + pdf_folder: str, + output_file: str, ) -> None: """ Generate a simple static HTML report of test results. @@ -153,7 +157,11 @@ def generate_html_report( elif test_type == "absent" and hasattr(test, "text"): text = getattr(test, "text", "") html += f"""

Text should not appear: "{text}"

\n""" - elif test_type == "order" and hasattr(test, "before") and hasattr(test, "after"): + elif ( + test_type == "order" + and hasattr(test, "before") + and hasattr(test, "after") + ): before = getattr(test, "before", "") after = getattr(test, "after", "") html += f"""

Text order: "{before}" should appear before "{after}"

\n""" @@ -198,9 +206,19 @@ def generate_html_report( md_content = None try: md_base = os.path.splitext(pdf_name)[0] - md_files = list(glob.glob(os.path.join(os.path.dirname(pdf_folder), candidate, f"{md_base}_pg{page}_repeat*.md"))) + md_files = list( + glob.glob( + os.path.join( + os.path.dirname(pdf_folder), + candidate, + f"{md_base}_pg{page}_repeat*.md", + ) + ) + ) if md_files: - md_file_path = md_files[0] # Use the first repeat as an example + md_file_path = md_files[ + 0 + ] # Use the first repeat as an example with open(md_file_path, "r", encoding="utf-8") as f: md_content = f.read() except Exception as e: diff --git a/benchmarks/olmocr/bench/runners/run_reducto.py b/benchmarks/olmocr/bench/runners/run_reducto.py index 5755a9e..df1db55 100644 --- a/benchmarks/olmocr/bench/runners/run_reducto.py +++ b/benchmarks/olmocr/bench/runners/run_reducto.py @@ -85,6 +85,7 @@ def run_reducto( ) try: from src.commons_reducto import record_usage + record_usage("parse", getattr(response, "usage", None)) except Exception: pass diff --git a/benchmarks/olmocr/bench/tests.py b/benchmarks/olmocr/bench/tests.py index 0045ea1..5a691c1 100644 --- a/benchmarks/olmocr/bench/tests.py +++ b/benchmarks/olmocr/bench/tests.py @@ -26,10 +26,18 @@ class TableData: """Class to hold table data and metadata about headers.""" data: np.ndarray # The actual table data - header_rows: Set[int] = field(default_factory=set) # Indices of rows that are headers - header_cols: Set[int] = field(default_factory=set) # Indices of columns that are headers - col_headers: dict = field(default_factory=dict) # Maps column index to header text, handling colspan - row_headers: dict = field(default_factory=dict) # Maps row index to header text, handling rowspan + header_rows: Set[int] = field( + default_factory=set + ) # Indices of rows that are headers + header_cols: Set[int] = field( + default_factory=set + ) # Indices of columns that are headers + col_headers: dict = field( + default_factory=dict + ) # Maps column index to header text, handling colspan + row_headers: dict = field( + default_factory=dict + ) # Maps row index to header text, handling rowspan def __repr__(self) -> str: """Returns a concise representation of the TableData object for debugging.""" @@ -40,7 +48,9 @@ def __str__(self) -> str: output = [] # Table dimensions - output.append(f"Table: {self.data.shape[0]} rows × {self.data.shape[1]} columns") + output.append( + f"Table: {self.data.shape[0]} rows × {self.data.shape[1]} columns" + ) # Header info output.append(f"Header rows: {sorted(self.header_rows)}") @@ -52,7 +62,11 @@ def __str__(self) -> str: # Add a header for row indices output.append(separator) headers = [""] + [f"Column {i}" for i in range(self.data.shape[1])] - output.append("| {:<5} | ".format("Row") + " | ".join(["{:<15}".format(h) for h in headers[1:]]) + " |") + output.append( + "| {:<5} | ".format("Row") + + " | ".join(["{:<15}".format(h) for h in headers[1:]]) + + " |" + ) output.append(separator) # Format each row @@ -68,7 +82,11 @@ def __str__(self) -> str: cell = f"*{cell}*" cells.append(cell) - row_str = "| {:<5} | ".format(i) + " | ".join(["{:<15}".format(c) for c in cells]) + " |" + row_str = ( + "| {:<5} | ".format(i) + + " | ".join(["{:<15}".format(c) for c in cells]) + + " |" + ) output.append(row_str) output.append(separator) @@ -137,7 +155,21 @@ def normalize_text(md_content: str) -> str: md_content = unicodedata.normalize("NFC", md_content) # Dictionary of characters to replace: keys are fancy characters, values are ASCII equivalents, unicode micro with greek mu comes up often enough too - replacements = {"‘": "'", "’": "'", "‚": "'", "“": '"', "”": '"', "„": '"', "＿": "_", "–": "-", "—": "-", "‑": "-", "‒": "-", "−": "-", "\u00b5": "\u03bc"} + replacements = { + "‘": "'", + "’": "'", + "‚": "'", + "“": '"', + "”": '"', + "„": '"', + "＿": "_", + "–": "-", + "—": "-", + "‑": "-", + "‒": "-", + "−": "-", + "\u00b5": "\u03bc", + } # Apply all replacements from the dictionary for fancy_char, ascii_char in replacements.items(): @@ -185,7 +217,9 @@ def parse_markdown_tables(md_content: str) -> List[TableData]: if table_data and len(table_data) > 0: # Convert to numpy array for easier manipulation max_cols = max(len(row) for row in table_data) - padded_data = [row + [""] * (max_cols - len(row)) for row in table_data] + padded_data = [ + row + [""] * (max_cols - len(row)) for row in table_data + ] table_array = np.array(padded_data) # In markdown tables, the first row is typically a header row @@ -196,20 +230,28 @@ def parse_markdown_tables(md_content: str) -> List[TableData]: if len(table_array) > 0: for col_idx in range(table_array.shape[1]): if col_idx < len(table_array[0]): - col_headers[col_idx] = [(0, table_array[0, col_idx])] + col_headers[col_idx] = [ + (0, table_array[0, col_idx]) + ] # Set up row_headers with first column headers for each row row_headers = {} if table_array.shape[1] > 0: - for row_idx in range(1, table_array.shape[0]): # Skip header row - row_headers[row_idx] = [(0, table_array[row_idx, 0])] # First column as heading + for row_idx in range( + 1, table_array.shape[0] + ): # Skip header row + row_headers[row_idx] = [ + (0, table_array[row_idx, 0]) + ] # First column as heading # Create TableData object parsed_tables.append( TableData( data=table_array, header_rows=header_rows, - header_cols={0} if table_array.shape[1] > 0 else set(), # First column as header + header_cols={0} + if table_array.shape[1] > 0 + else set(), # First column as header col_headers=col_headers, row_headers=row_headers, ) @@ -239,14 +281,18 @@ def parse_markdown_tables(md_content: str) -> List[TableData]: row_headers = {} if table_array.shape[1] > 0: for row_idx in range(1, table_array.shape[0]): # Skip header row - row_headers[row_idx] = [(0, table_array[row_idx, 0])] # First column as heading + row_headers[row_idx] = [ + (0, table_array[row_idx, 0]) + ] # First column as heading # Create TableData object parsed_tables.append( TableData( data=table_array, header_rows=header_rows, - header_cols={0} if table_array.shape[1] > 0 else set(), # First column as header + header_cols={0} + if table_array.shape[1] > 0 + else set(), # First column as header col_headers=col_headers, row_headers=row_headers, ) @@ -374,7 +420,9 @@ def parse_html_tables(html_content: str) -> List[TableData]: if j == 0 and i > 0: # Only for cells directly below cell_grid[(row_idx + i, col_idx + j)] = cell_text else: - cell_grid[(row_idx + i, col_idx + j)] = "" # Mark other spans as empty + cell_grid[(row_idx + i, col_idx + j)] = ( + "" # Mark other spans as empty + ) # If this is a header cell (th), mark it and its span if cell.name == "th": @@ -427,12 +475,19 @@ def parse_html_tables(html_content: str) -> List[TableData]: # Add this header to all columns it spans over for row_idx in range(len(table_data)): if row_idx not in header_rows: # Only apply to data rows - for j in range(col, len(table_data[row_idx]) if row_idx < len(table_data) else 0): + for j in range( + col, + len(table_data[row_idx]) + if row_idx < len(table_data) + else 0, + ): # Add header info to data cells in these columns if j not in col_headers: col_headers[j] = [] if not any(h[1] == header_text for h in col_headers[j]): - header_row = min([r for r, t in col_headers.get(col, [(0, "")])]) + header_row = min( + [r for r, t in col_headers.get(col, [(0, "")])] + ) col_headers[j].append((header_row, header_text)) # Handle row headers @@ -452,7 +507,9 @@ def parse_html_tables(html_content: str) -> List[TableData]: if col_idx < len(row) and row[col_idx].strip(): if row_idx not in row_headers: row_headers[row_idx] = [] - if not any(h[1] == row[col_idx] for h in row_headers.get(row_idx, [])): + if not any( + h[1] == row[col_idx] for h in row_headers.get(row_idx, []) + ): row_headers[row_idx].append((col_idx, row[col_idx])) # Calculate max columns for padding @@ -465,7 +522,13 @@ def parse_html_tables(html_content: str) -> List[TableData]: # Create TableData object with the table and header information parsed_tables.append( - TableData(data=table_array, header_rows=header_rows, header_cols=header_cols, col_headers=col_headers, row_headers=row_headers) + TableData( + data=table_array, + header_rows=header_rows, + header_cols=header_cols, + col_headers=col_headers, + row_headers=row_headers, + ) ) return parsed_tables @@ -556,20 +619,28 @@ def run(self, md_content: str) -> Tuple[bool, str]: md_content = md_content[-self.last_n :] # Threshold for fuzzy matching derived from max_diffs - threshold = 1.0 - (self.max_diffs / (len(reference_query) if len(reference_query) > 0 else 1)) + threshold = 1.0 - ( + self.max_diffs / (len(reference_query) if len(reference_query) > 0 else 1) + ) best_ratio = fuzz.partial_ratio(reference_query, md_content) / 100.0 if self.type == TestType.PRESENT.value: if best_ratio >= threshold: return True, "" else: - msg = f"Expected '{reference_query[:40]}...' with threshold {threshold} " f"but best match ratio was {best_ratio:.3f}" + msg = ( + f"Expected '{reference_query[:40]}...' with threshold {threshold} " + f"but best match ratio was {best_ratio:.3f}" + ) return False, msg else: # ABSENT if best_ratio < threshold: return True, "" else: - msg = f"Expected absence of '{reference_query[:40]}...' with threshold {threshold} " f"but best match ratio was {best_ratio:.3f}" + msg = ( + f"Expected absence of '{reference_query[:40]}...' with threshold {threshold} " + f"but best match ratio was {best_ratio:.3f}" + ) return False, msg @@ -596,25 +667,43 @@ def __post_init__(self): raise ValidationError("Before field cannot be empty") if not self.after.strip(): raise ValidationError("After field cannot be empty") - if self.max_diffs > len(self.before) // 2 or self.max_diffs > len(self.after) // 2: - raise ValidationError("Max diffs is too large for this test, greater than 50% of the search string") + if ( + self.max_diffs > len(self.before) // 2 + or self.max_diffs > len(self.after) // 2 + ): + raise ValidationError( + "Max diffs is too large for this test, greater than 50% of the search string" + ) def run(self, md_content: str) -> Tuple[bool, str]: md_content = normalize_text(md_content) - before_matches = find_near_matches(self.before, md_content, max_l_dist=self.max_diffs) - after_matches = find_near_matches(self.after, md_content, max_l_dist=self.max_diffs) + before_matches = find_near_matches( + self.before, md_content, max_l_dist=self.max_diffs + ) + after_matches = find_near_matches( + self.after, md_content, max_l_dist=self.max_diffs + ) if not before_matches: - return False, f"'before' text '{self.before[:40]}...' not found with max_l_dist {self.max_diffs}" + return ( + False, + f"'before' text '{self.before[:40]}...' not found with max_l_dist {self.max_diffs}", + ) if not after_matches: - return False, f"'after' text '{self.after[:40]}...' not found with max_l_dist {self.max_diffs}" + return ( + False, + f"'after' text '{self.after[:40]}...' not found with max_l_dist {self.max_diffs}", + ) for before_match in before_matches: for after_match in after_matches: if before_match.start < after_match.start: return True, "" - return False, (f"Could not find a location where '{self.before[:40]}...' appears before " f"'{self.after[:40]}...'.") + return False, ( + f"Could not find a location where '{self.before[:40]}...' appears before " + f"'{self.after[:40]}...'." + ) @dataclass @@ -671,7 +760,9 @@ def run(self, content: str) -> Tuple[bool, str]: failed_reasons = [] # Threshold for fuzzy matching derived from max_diffs - threshold = 1.0 - (self.max_diffs / (len(self.cell) if len(self.cell) > 0 else 1)) + threshold = 1.0 - ( + self.max_diffs / (len(self.cell) if len(self.cell) > 0 else 1) + ) threshold = max(0.5, threshold) # Parse tables based on content_type @@ -716,23 +807,45 @@ def run(self, content: str) -> Tuple[bool, str]: if self.up and row_idx > 0: up_cell = normalize_text(table_array[row_idx - 1, col_idx]) up_similarity = fuzz.ratio(self.up, up_cell) / 100.0 - if up_similarity < max(0.5, 1.0 - (self.max_diffs / (len(self.up) if len(self.up) > 0 else 1))): + if up_similarity < max( + 0.5, + 1.0 + - (self.max_diffs / (len(self.up) if len(self.up) > 0 else 1)), + ): all_relationships_satisfied = False - current_failed_reasons.append(f"Cell above '{up_cell}' doesn't match expected '{self.up}' (similarity: {up_similarity:.2f})") + current_failed_reasons.append( + f"Cell above '{up_cell}' doesn't match expected '{self.up}' (similarity: {up_similarity:.2f})" + ) # Check down relationship if self.down and row_idx < table_array.shape[0] - 1: down_cell = normalize_text(table_array[row_idx + 1, col_idx]) down_similarity = fuzz.ratio(self.down, down_cell) / 100.0 - if down_similarity < max(0.5, 1.0 - (self.max_diffs / (len(self.down) if len(self.down) > 0 else 1))): + if down_similarity < max( + 0.5, + 1.0 + - ( + self.max_diffs + / (len(self.down) if len(self.down) > 0 else 1) + ), + ): all_relationships_satisfied = False - current_failed_reasons.append(f"Cell below '{down_cell}' doesn't match expected '{self.down}' (similarity: {down_similarity:.2f})") + current_failed_reasons.append( + f"Cell below '{down_cell}' doesn't match expected '{self.down}' (similarity: {down_similarity:.2f})" + ) # Check left relationship if self.left and col_idx > 0: left_cell = normalize_text(table_array[row_idx, col_idx - 1]) left_similarity = fuzz.ratio(self.left, left_cell) / 100.0 - if left_similarity < max(0.5, 1.0 - (self.max_diffs / (len(self.left) if len(self.left) > 0 else 1))): + if left_similarity < max( + 0.5, + 1.0 + - ( + self.max_diffs + / (len(self.left) if len(self.left) > 0 else 1) + ), + ): all_relationships_satisfied = False current_failed_reasons.append( f"Cell to the left '{left_cell}' doesn't match expected '{self.left}' (similarity: {left_similarity:.2f})" @@ -742,7 +855,14 @@ def run(self, content: str) -> Tuple[bool, str]: if self.right and col_idx < table_array.shape[1] - 1: right_cell = normalize_text(table_array[row_idx, col_idx + 1]) right_similarity = fuzz.ratio(self.right, right_cell) / 100.0 - if right_similarity < max(0.5, 1.0 - (self.max_diffs / (len(self.right) if len(self.right) > 0 else 1))): + if right_similarity < max( + 0.5, + 1.0 + - ( + self.max_diffs + / (len(self.right) if len(self.right) > 0 else 1) + ), + ): all_relationships_satisfied = False current_failed_reasons.append( f"Cell to the right '{right_cell}' doesn't match expected '{self.right}' (similarity: {right_similarity:.2f})" @@ -759,11 +879,24 @@ def run(self, content: str) -> Tuple[bool, str]: if col_idx in table_data.col_headers: for _, header_text in table_data.col_headers[col_idx]: header_text = normalize_text(header_text) - similarity = fuzz.ratio(self.top_heading, header_text) / 100.0 + similarity = ( + fuzz.ratio(self.top_heading, header_text) / 100.0 + ) if similarity > best_similarity: best_similarity = similarity best_match = header_text - if best_similarity >= max(0.5, 1.0 - (self.max_diffs / (len(self.top_heading) if len(self.top_heading) > 0 else 1))): + if best_similarity >= max( + 0.5, + 1.0 + - ( + self.max_diffs + / ( + len(self.top_heading) + if len(self.top_heading) > 0 + else 1 + ) + ), + ): top_heading_found = True break @@ -772,11 +905,24 @@ def run(self, content: str) -> Tuple[bool, str]: for i in sorted(header_rows): if i < row_idx and table_array[i, col_idx].strip(): header_text = normalize_text(table_array[i, col_idx]) - similarity = fuzz.ratio(self.top_heading, header_text) / 100.0 + similarity = ( + fuzz.ratio(self.top_heading, header_text) / 100.0 + ) if similarity > best_similarity: best_similarity = similarity best_match = header_text - if best_similarity >= max(0.5, 1.0 - (self.max_diffs / (len(self.top_heading) if len(self.top_heading) > 0 else 1))): + if best_similarity >= max( + 0.5, + 1.0 + - ( + self.max_diffs + / ( + len(self.top_heading) + if len(self.top_heading) > 0 + else 1 + ) + ), + ): top_heading_found = True break @@ -785,15 +931,30 @@ def run(self, content: str) -> Tuple[bool, str]: for i in range(row_idx): if table_array[i, col_idx].strip(): header_text = normalize_text(table_array[i, col_idx]) - similarity = fuzz.ratio(self.top_heading, header_text) / 100.0 + similarity = ( + fuzz.ratio(self.top_heading, header_text) / 100.0 + ) if similarity > best_similarity: best_similarity = similarity best_match = header_text if not best_match: all_relationships_satisfied = False - current_failed_reasons.append(f"No top heading found for cell at ({row_idx}, {col_idx})") - elif best_similarity < max(0.5, 1.0 - (self.max_diffs / (len(self.top_heading) if len(self.top_heading) > 0 else 1))): + current_failed_reasons.append( + f"No top heading found for cell at ({row_idx}, {col_idx})" + ) + elif best_similarity < max( + 0.5, + 1.0 + - ( + self.max_diffs + / ( + len(self.top_heading) + if len(self.top_heading) > 0 + else 1 + ) + ), + ): all_relationships_satisfied = False current_failed_reasons.append( f"Top heading '{best_match}' doesn't match expected '{self.top_heading}' (similarity: {best_similarity:.2f})" @@ -810,11 +971,24 @@ def run(self, content: str) -> Tuple[bool, str]: if row_idx in table_data.row_headers: for _, header_text in table_data.row_headers[row_idx]: header_text = normalize_text(header_text) - similarity = fuzz.ratio(self.left_heading, header_text) / 100.0 + similarity = ( + fuzz.ratio(self.left_heading, header_text) / 100.0 + ) if similarity > best_similarity: best_similarity = similarity best_match = header_text - if best_similarity >= max(0.5, 1.0 - (self.max_diffs / (len(self.left_heading) if len(self.left_heading) > 0 else 1))): + if best_similarity >= max( + 0.5, + 1.0 + - ( + self.max_diffs + / ( + len(self.left_heading) + if len(self.left_heading) > 0 + else 1 + ) + ), + ): left_heading_found = True break @@ -823,11 +997,24 @@ def run(self, content: str) -> Tuple[bool, str]: for j in sorted(header_cols): if j < col_idx and table_array[row_idx, j].strip(): header_text = normalize_text(table_array[row_idx, j]) - similarity = fuzz.ratio(self.left_heading, header_text) / 100.0 + similarity = ( + fuzz.ratio(self.left_heading, header_text) / 100.0 + ) if similarity > best_similarity: best_similarity = similarity best_match = header_text - if best_similarity >= max(0.5, 1.0 - (self.max_diffs / (len(self.left_heading) if len(self.left_heading) > 0 else 1))): + if best_similarity >= max( + 0.5, + 1.0 + - ( + self.max_diffs + / ( + len(self.left_heading) + if len(self.left_heading) > 0 + else 1 + ) + ), + ): left_heading_found = True break @@ -836,15 +1023,30 @@ def run(self, content: str) -> Tuple[bool, str]: for j in range(col_idx): if table_array[row_idx, j].strip(): header_text = normalize_text(table_array[row_idx, j]) - similarity = fuzz.ratio(self.left_heading, header_text) / 100.0 + similarity = ( + fuzz.ratio(self.left_heading, header_text) / 100.0 + ) if similarity > best_similarity: best_similarity = similarity best_match = header_text if not best_match: all_relationships_satisfied = False - current_failed_reasons.append(f"No left heading found for cell at ({row_idx}, {col_idx})") - elif best_similarity < max(0.5, 1.0 - (self.max_diffs / (len(self.left_heading) if len(self.left_heading) > 0 else 1))): + current_failed_reasons.append( + f"No left heading found for cell at ({row_idx}, {col_idx})" + ) + elif best_similarity < max( + 0.5, + 1.0 + - ( + self.max_diffs + / ( + len(self.left_heading) + if len(self.left_heading) > 0 + else 1 + ) + ), + ): all_relationships_satisfied = False current_failed_reasons.append( f"Left heading '{best_match}' doesn't match expected '{self.left_heading}' (similarity: {best_similarity:.2f})" @@ -858,9 +1060,15 @@ def run(self, content: str) -> Tuple[bool, str]: # If we've gone through all tables and all matching cells and none satisfied all relationships if not failed_reasons: - return False, f"No cell matching '{self.cell}' found in any table with threshold {threshold}" + return ( + False, + f"No cell matching '{self.cell}' found in any table with threshold {threshold}", + ) else: - return False, f"Found cells matching '{self.cell}' but relationships were not satisfied: {'; '.join(failed_reasons)}" + return ( + False, + f"Found cells matching '{self.cell}' but relationships were not satisfied: {'; '.join(failed_reasons)}", + ) @dataclass @@ -887,10 +1095,15 @@ def run(self, content: str) -> Tuple[bool, str]: if self.max_length_skips_image_alt_tags: # Remove markdown image tags like ![alt text](image.png) from the text length count content_for_length_check = re.sub(r"!\[.*?\]\(.*?\)", "", content) - base_content_len = len("".join(c for c in content_for_length_check if c.isalnum()).strip()) + base_content_len = len( + "".join(c for c in content_for_length_check if c.isalnum()).strip() + ) if base_content_len > self.max_length: - return False, f"{base_content_len} characters were output for a page we expected to be blank" + return ( + False, + f"{base_content_len} characters were output for a page we expected to be blank", + ) else: return True, "" @@ -906,7 +1119,10 @@ def run(self, content: str) -> Tuple[bool, str]: for index, count in enumerate(repeats): if count > self.max_repeats: - return False, f"Text ends with {count} repeating {index+1}-grams, invalid" + return ( + False, + f"Text ends with {count} repeating {index + 1}-grams, invalid", + ) pattern = re.compile( r"[" @@ -970,7 +1186,9 @@ def run(self, content: str) -> Tuple[bool, str]: equations.extend([e.strip() for e in matches]) # Replace all instances of this pattern with empty strings - modified_content = re.sub(replace_pattern, "", modified_content, flags=re.DOTALL) + modified_content = re.sub( + replace_pattern, "", modified_content, flags=re.DOTALL + ) # If an equation in the markdown exactly matches our math string, then that's good enough # we don't have to do a more expensive comparison @@ -1046,7 +1264,9 @@ def load_tests(jsonl_file: str) -> List[BasePDFTest]: A list of test objects. """ - def process_line_with_number(line_tuple: Tuple[int, str]) -> Optional[Tuple[int, BasePDFTest]]: + def process_line_with_number( + line_tuple: Tuple[int, str], + ) -> Optional[Tuple[int, BasePDFTest]]: """ Process a single line from the JSONL file and return a tuple of (line_number, test object). Returns None for empty lines. @@ -1078,9 +1298,13 @@ def process_line_with_number(line_tuple: Tuple[int, str]) -> Optional[Tuple[int, # Use a ThreadPoolExecutor to process each line in parallel. with ThreadPoolExecutor(max_workers=min(os.cpu_count() or 1, 64)) as executor: # Submit all tasks concurrently. - futures = {executor.submit(process_line_with_number, item): item[0] for item in lines} + futures = { + executor.submit(process_line_with_number, item): item[0] for item in lines + } # Use tqdm to show progress as futures complete. - for future in tqdm(as_completed(futures), total=len(futures), desc="Loading tests"): + for future in tqdm( + as_completed(futures), total=len(futures), desc="Loading tests" + ): result = future.result() if result is not None: _, test = result @@ -1090,7 +1314,9 @@ def process_line_with_number(line_tuple: Tuple[int, str]) -> Optional[Tuple[int, unique_ids = set() for test in tests: if test.id in unique_ids: - raise ValidationError(f"Test with duplicate id {test.id} found, error loading tests.") + raise ValidationError( + f"Test with duplicate id {test.id} found, error loading tests." + ) unique_ids.add(test.id) return tests diff --git a/benchmarks/olmocr/bench/utils.py b/benchmarks/olmocr/bench/utils.py index 5fe5f38..768b8fe 100644 --- a/benchmarks/olmocr/bench/utils.py +++ b/benchmarks/olmocr/bench/utils.py @@ -3,7 +3,12 @@ import numpy as np -def calculate_bootstrap_ci(test_scores: List[float], n_bootstrap: int = 1000, ci_level: float = 0.95, splits: List[int] = None) -> Tuple[float, float]: +def calculate_bootstrap_ci( + test_scores: List[float], + n_bootstrap: int = 1000, + ci_level: float = 0.95, + splits: List[int] = None, +) -> Tuple[float, float]: """ Calculate bootstrap confidence interval for test scores, respecting category splits. @@ -35,7 +40,9 @@ def calculate_bootstrap_ci(test_scores: List[float], n_bootstrap: int = 1000, ci else: # Validate splits if sum(splits) != len(scores): - raise ValueError(f"Sum of splits ({sum(splits)}) must equal length of test_scores ({len(scores)})") + raise ValueError( + f"Sum of splits ({sum(splits)}) must equal length of test_scores ({len(scores)})" + ) # Convert flat scores list to a list of category scores category_scores = [] @@ -52,7 +59,9 @@ def calculate_bootstrap_ci(test_scores: List[float], n_bootstrap: int = 1000, ci for cat_scores in category_scores: if len(cat_scores) > 0: # Sample with replacement within this category - cat_sample = np.random.choice(cat_scores, size=len(cat_scores), replace=True) + cat_sample = np.random.choice( + cat_scores, size=len(cat_scores), replace=True + ) category_means.append(np.mean(cat_sample)) # Overall score is average of category means (if any categories have scores) @@ -68,7 +77,11 @@ def calculate_bootstrap_ci(test_scores: List[float], n_bootstrap: int = 1000, ci def perform_permutation_test( - scores_a: List[float], scores_b: List[float], n_permutations: int = 10000, splits_a: List[int] = None, splits_b: List[int] = None + scores_a: List[float], + scores_b: List[float], + n_permutations: int = 10000, + splits_a: List[int] = None, + splits_b: List[int] = None, ) -> Tuple[float, float]: """ Perform a permutation test to determine if there's a significant difference @@ -133,9 +146,13 @@ def mean_of_category_means(scores, splits=None): # For category-based permutation test, we need to maintain category structure # Validate that the splits match the score lengths if splits_a is not None and sum(splits_a) != len(scores_a): - raise ValueError(f"Sum of splits_a ({sum(splits_a)}) must equal length of scores_a ({len(scores_a)})") + raise ValueError( + f"Sum of splits_a ({sum(splits_a)}) must equal length of scores_a ({len(scores_a)})" + ) if splits_b is not None and sum(splits_b) != len(scores_b): - raise ValueError(f"Sum of splits_b ({sum(splits_b)}) must equal length of scores_b ({len(scores_b)})") + raise ValueError( + f"Sum of splits_b ({sum(splits_b)}) must equal length of scores_b ({len(scores_b)})" + ) # Create category structures categories_a = [] diff --git a/benchmarks/olmocr/data/renderpdf.py b/benchmarks/olmocr/data/renderpdf.py index d33a7ca..0c77a6a 100644 --- a/benchmarks/olmocr/data/renderpdf.py +++ b/benchmarks/olmocr/data/renderpdf.py @@ -6,7 +6,9 @@ from PIL import Image -def get_pdf_media_box_width_height(local_pdf_path: str, page_num: int) -> tuple[float, float]: +def get_pdf_media_box_width_height( + local_pdf_path: str, page_num: int +) -> tuple[float, float]: """ Get the MediaBox dimensions for a specific page in a PDF file using the pdfinfo command. @@ -15,10 +17,22 @@ def get_pdf_media_box_width_height(local_pdf_path: str, page_num: int) -> tuple[ :return: A dictionary containing MediaBox dimensions or None if not found """ # Construct the pdfinfo command to extract info for the specific page - command = ["pdfinfo", "-f", str(page_num), "-l", str(page_num), "-box", "-enc", "UTF-8", local_pdf_path] + command = [ + "pdfinfo", + "-f", + str(page_num), + "-l", + str(page_num), + "-box", + "-enc", + "UTF-8", + local_pdf_path, + ] # Run the command using subprocess - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + result = subprocess.run( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) # Check if there is any error in executing the command if result.returncode != 0: @@ -36,7 +50,9 @@ def get_pdf_media_box_width_height(local_pdf_path: str, page_num: int) -> tuple[ raise ValueError("MediaBox not found in the PDF info.") -def render_pdf_to_base64png(local_pdf_path: str, page_num: int, target_longest_image_dim: int = 2048) -> str: +def render_pdf_to_base64png( + local_pdf_path: str, page_num: int, target_longest_image_dim: int = 2048 +) -> str: longest_dim = max(get_pdf_media_box_width_height(local_pdf_path, page_num)) # Convert PDF page to PNG using pdftoppm @@ -49,7 +65,9 @@ def render_pdf_to_base64png(local_pdf_path: str, page_num: int, target_longest_i "-l", str(page_num), "-r", - str(target_longest_image_dim * 72 / longest_dim), # 72 pixels per point is the conversion factor + str( + target_longest_image_dim * 72 / longest_dim + ), # 72 pixels per point is the conversion factor local_pdf_path, ], timeout=120, @@ -60,7 +78,9 @@ def render_pdf_to_base64png(local_pdf_path: str, page_num: int, target_longest_i return base64.b64encode(pdftoppm_result.stdout).decode("utf-8") -def render_pdf_to_base64webp(local_pdf_path: str, page: int, target_longest_image_dim: int = 1024): +def render_pdf_to_base64webp( + local_pdf_path: str, page: int, target_longest_image_dim: int = 1024 +): base64_png = render_pdf_to_base64png(local_pdf_path, page, target_longest_image_dim) png_image = Image.open(io.BytesIO(base64.b64decode(base64_png))) diff --git a/benchmarks/olmocr/olmocr_bench_reducto.py b/benchmarks/olmocr/olmocr_bench_reducto.py index d107e7d..82a4a28 100644 --- a/benchmarks/olmocr/olmocr_bench_reducto.py +++ b/benchmarks/olmocr/olmocr_bench_reducto.py @@ -85,8 +85,7 @@ def download_full_dataset(): tests = [json.loads(l) for l in f if l.strip()] data_dir.mkdir(parents=True, exist_ok=True) with open(jsonl_dest, "w") as f: - for t in tests: - f.write(json.dumps(t) + "\n") + f.writelines(json.dumps(t) + "\n" for t in tests) print(f" {split}: {len(tests)} tests") for t in tests: all_pdfs.add(t["pdf"]) @@ -137,7 +136,6 @@ async def generate_outputs(data_dir: Path): pdf_folder = data_dir / "pdfs" output_folder = data_dir / CANDIDATE_NAME - pdf_pages = set() for jsonl_file in data_dir.glob("*.jsonl"): with open(jsonl_file) as f: diff --git a/benchmarks/olmocr/repeatdetect.py b/benchmarks/olmocr/repeatdetect.py index 03dcbe5..8ea5f2c 100644 --- a/benchmarks/olmocr/repeatdetect.py +++ b/benchmarks/olmocr/repeatdetect.py @@ -168,7 +168,7 @@ def testLargeRandom(self): end = time.perf_counter() - print(f"testLargeRandom took {end-start:0.0001f} seconds") + print(f"testLargeRandom took {end - start:0.0001f} seconds") if __name__ == "__main__": diff --git a/benchmarks/spider2_lite/fetch_data.py b/benchmarks/spider2_lite/fetch_data.py index 2a8f1b4..5efdb91 100644 --- a/benchmarks/spider2_lite/fetch_data.py +++ b/benchmarks/spider2_lite/fetch_data.py @@ -39,7 +39,10 @@ def ensure_repo() -> None: print(f"[clone] xlang-ai/Spider2 (depth=1) -> {REPO_DIR}") subprocess.run( [ - "git", "clone", "--depth", "1", + "git", + "clone", + "--depth", + "1", "https://github.com/xlang-ai/Spider2.git", str(REPO_DIR), ], @@ -58,7 +61,10 @@ def ensure_sqlite_dbs() -> None: try: import gdown except ImportError: - print("ERROR: `gdown` is required to download from Google Drive.", file=sys.stderr) + print( + "ERROR: `gdown` is required to download from Google Drive.", + file=sys.stderr, + ) print(" Add it with `uv add gdown` and re-run.", file=sys.stderr) sys.exit(1) print(f"[download] Google Drive id={SQLITE_ZIP_DRIVE_ID} -> {zip_path}") diff --git a/pyproject.toml b/pyproject.toml index aba7e9e..9b2341b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,13 +42,12 @@ dependencies = [ "olmocr==0.4.27", "pymupdf>=1.27.2", "jiwer>=3.0.0", - # olmOCR-bench scoring: fuzzy text matching + headless KaTeX equation render - # (needs `uv run python -m playwright install chromium` once). "fuzzysearch>=0.7.3", "playwright>=1.44.0", "gdown>=6.1.0", "google-genai>=2.18.1", "anthropic>=0.122.0", + "reducto>=0.22.0", ] [dependency-groups] @@ -60,3 +59,13 @@ dev = [ [tool.pytest.ini_options] asyncio_mode = "auto" testpaths = ["tests"] + +[tool.ruff] +extend-exclude = [ + "benchmarks/ocrbench_v2/eval_scripts", + "benchmarks/olmocr/bench", + "benchmarks/olmocr/data", +] + +[tool.ruff.lint.per-file-ignores] +"benchmarks/olmocr/olmocr_bench_reducto.py" = ["BLE001", "ASYNC230"] diff --git a/scripts/plot_inkling_radar.py b/scripts/plot_inkling_radar.py index 2713456..2a09ef1 100644 --- a/scripts/plot_inkling_radar.py +++ b/scripts/plot_inkling_radar.py @@ -75,7 +75,6 @@ def j(name): return json.loads(p.read_text()) if p.exists() else None ocr = j("ocrbench_v2_fireworks_inkling_metrics.json") - olm = j("olmocr_fireworks_inkling_metrics.json") gpqa = j("fireworks_inkling_thinkingoff_gpqa_diamond_metrics.json") mmmu = j( "mmmupro_standard_fireworks_accounts-fireworks-models-inkling_reasoningoff_metrics.json" diff --git a/scripts/sheet_columns.py b/scripts/sheet_columns.py index d28665e..2e9f2b3 100644 --- a/scripts/sheet_columns.py +++ b/scripts/sheet_columns.py @@ -48,8 +48,11 @@ def main(): "OCRBench V2", f"{ocr['en_overall'] * 100:.2f}", "", - "Inkling = EN overall, n=10000. Sheet row says Thinking OFF; Inkling cannot " - "disable thinking, this is reasoning_effort=none (its floor). Small: not measured.", + ( + "Inkling = EN overall, n=10000. Sheet row says Thinking OFF; Inkling " + "cannot disable thinking, this is reasoning_effort=none (its floor). " + "Small: not measured." + ), ), ( "olmOCR", @@ -61,16 +64,20 @@ def main(): "RefCoco", "", "", - "BLANK ON PURPOSE: this row is TestA; our run is val " - f"(strict {32.37:.2f}, format-tolerant oracle {81.19:.2f}). Not the same split — " - "say the word and I'll run TestA to fill it properly.", + ( + "BLANK ON PURPOSE: this row is TestA; our run is val " + f"(strict {32.37:.2f}, format-tolerant oracle {81.19:.2f}). Not the " + "same split — say the word and I'll run TestA to fill it properly." + ), ), ( "VoxPoppuliCleaned-AA", f"{(1 - asr['corpus_wer']) * 100:.2f}", "", - "Reported as 1-WER to match this row's scale (corpus WER 4.61%, n=628). " - "Small: not measured.", + ( + "Reported as 1-WER to match this row's scale (corpus WER 4.61%, " + "n=628). Small: not measured." + ), ), ( "TheSOB", @@ -82,31 +89,40 @@ def main(): "Spider-2.0-lite", f"{spider['accuracy_of_local_135'] * 100:.2f}", "", - "64/135 SQLite subset. 5 of the 71 misses hit a 120s per-query cap added to the " - "scorer. Small: not measured.", + ( + "64/135 SQLite subset. 5 of the 71 misses hit a 120s per-query cap " + "added to the scorer. Small: not measured." + ), ), ( "GPQA-Diamond", f"{gpqa['accuracy'] * 100:.2f}", "88.30", - "CONFIG MISMATCH: this row is Thinking On; our Inkling run is the floor " - "(mean 6537 reasoning tokens/sample even so). Small 88.30 is Thinking Machines' " - "PUBLISHED figure at effort=0.99, NOT our measurement.", + ( + "CONFIG MISMATCH: this row is Thinking On; our Inkling run is the " + "floor (mean 6537 reasoning tokens/sample even so). Small 88.30 is " + "Thinking Machines' PUBLISHED figure at effort=0.99, NOT our measurement." + ), ), ( "MMMLU", f"{mmmlu['macro_accuracy'] * 100:.2f}", "", - "CONFIG MISMATCH: row is Thinking On; ours is the floor. Macro over 14 langs, " - "n=19950. Small: not measured.", + ( + "CONFIG MISMATCH: row is Thinking On; ours is the floor. Macro over " + "14 langs, n=19950. Small: not measured." + ), ), ( "MMMU-Pro", f"{mmmu_composite:.2f}", "73.10", - f"Inkling = standard+vision composite ({mmmu_std['accuracy'] * 100:.2f} / " - f"{mmmu_vis['accuracy'] * 100:.2f}), the published convention. Small 73.10 is the " - "PUBLISHED standard-setting figure, not measured, and not a composite.", + ( + f"Inkling = standard+vision composite ({mmmu_std['accuracy'] * 100:.2f}" + f" / {mmmu_vis['accuracy'] * 100:.2f}), the published convention. Small " + "73.10 is the PUBLISHED standard-setting figure, not measured, and not " + "a composite." + ), ), ] @@ -122,8 +138,6 @@ def main(): print(f"\nwrote {OUT}") - - # --------------------------------------------------------------------------- # Extra tabs. The "running" tab above is one column per model; these tabs are # one ROW per model, with sub-scores as columns. Emitted as separate TSV blocks @@ -136,7 +150,9 @@ def main(): def emit_tabs(): ocr = j("ocrbench_v2_fireworks_inkling_metrics.json") gpqa = j("fireworks_inkling_thinkingoff_gpqa_diamond_metrics.json") - mmmlu = j("mmmlulite_fireworks_accounts-fireworks-models-inkling_reasoningoff_metrics.json") + mmmlu = j( + "mmmlulite_fireworks_accounts-fireworks-models-inkling_reasoningoff_metrics.json" + ) spider = j("spider2_lite_local_fireworks_inkling_metrics.json") en = ocr["en_scores"] @@ -148,46 +164,124 @@ def p(x): # --- OCRBench V2 tab: model | Avg Real | Recog | Refer | Spot | Extract # | Parse | Calc | Understand | Reason (EN scores; matches the tab's # Interfaze row, whose Avg Real equals the running-tab OCRBench value) - blocks.append(("OCRBench V2 tab", - ["model", "Avg Real", "Recog", "Refer", "Spot", "Extract", "Parse", - "Calc", "Understand", "Reason"], - ["Inkling", p(ocr["en_overall"]), - p(en["text_recognition"]["avg"]), p(en["text_detection"]["avg"]), - p(en["text_spotting"]["avg"]), p(en["relationship_extraction"]["avg"]), - p(en["element_parsing"]["avg"]), p(en["mathematical_calculation"]["avg"]), - p(en["visual_text_understanding"]["avg"]), p(en["knowledge_reasoning"]["avg"])])) + blocks.append( + ( + "OCRBench V2 tab", + [ + "model", + "Avg Real", + "Recog", + "Refer", + "Spot", + "Extract", + "Parse", + "Calc", + "Understand", + "Reason", + ], + [ + "Inkling", + p(ocr["en_overall"]), + p(en["text_recognition"]["avg"]), + p(en["text_detection"]["avg"]), + p(en["text_spotting"]["avg"]), + p(en["relationship_extraction"]["avg"]), + p(en["element_parsing"]["avg"]), + p(en["mathematical_calculation"]["avg"]), + p(en["visual_text_understanding"]["avg"]), + p(en["knowledge_reasoning"]["avg"]), + ], + ) + ) # --- olmOCR tab. olmOCR-bench prints its per-split table to stdout instead # of persisting a metrics file, so these come from the run log # (logs/fireworks_inkling/olmocr.log, 1403/1403 pages, 8413 tests). - blocks.append(("olmOCR tab", - ["model", "Overall", "ArXiv", "OldScansMath", "Tables", "OldScans", - "Headers", "MultiCol", "LongTinyText", "Base", "Real Overall"], - ["Inkling", "74.90", "71.20", "78.80", "82.60", "41.60", "91.60", - "72.40", "61.80", "99.10", "74.9±1.1"])) + blocks.append( + ( + "olmOCR tab", + [ + "model", + "Overall", + "ArXiv", + "OldScansMath", + "Tables", + "OldScans", + "Headers", + "MultiCol", + "LongTinyText", + "Base", + "Real Overall", + ], + [ + "Inkling", + "74.90", + "71.20", + "78.80", + "82.60", + "41.60", + "91.60", + "72.40", + "61.80", + "99.10", + "74.9±1.1", + ], + ) + ) # --- GPQA tab: model | Overall | Physics (n=86) | Chemistry (n=93) | Biology (n=19) d = gpqa["per_domain"] - blocks.append(("GPQA tab", - ["model", "Overall", "Physics (n=86)", "Chemistry (n=93)", "Biology (n=19)"], - ["Inkling", p(gpqa["accuracy"]), p(d["Physics"]["accuracy"]), - p(d["Chemistry"]["accuracy"]), p(d["Biology"]["accuracy"])])) + blocks.append( + ( + "GPQA tab", + [ + "model", + "Overall", + "Physics (n=86)", + "Chemistry (n=93)", + "Biology (n=19)", + ], + [ + "Inkling", + p(gpqa["accuracy"]), + p(d["Physics"]["accuracy"]), + p(d["Chemistry"]["accuracy"]), + p(d["Biology"]["accuracy"]), + ], + ) + ) # --- MMMLU tab: languages are ROWS and models are COLUMNS, so this is a # column (macro first, then the 14 languages in the sheet's order). - order = ["FR_FR", "PT_BR", "BN_BD", "JA_JP", "DE_DE", "YO_NG", "ES_LA", - "ID_ID", "ZH_CN", "SW_KE", "IT_IT", "AR_XY", "KO_KR", "HI_IN"] + order = [ + "FR_FR", + "PT_BR", + "BN_BD", + "JA_JP", + "DE_DE", + "YO_NG", + "ES_LA", + "ID_ID", + "ZH_CN", + "SW_KE", + "IT_IT", + "AR_XY", + "KO_KR", + "HI_IN", + ] pl = mmmlu["per_language"] - blocks.append(("MMMLU tab (column: macro then langs)", - ["lang", "Inkling"], - None)) + blocks.append(("MMMLU tab (column: macro then langs)", ["lang", "Inkling"], None)) mmmlu_rows = [["macro", p(mmmlu["macro_accuracy"])]] mmmlu_rows += [[lg, p(pl[lg]["accuracy"])] for lg in order if lg in pl] # --- Spider-2.0 tab: Model | Spider-2.0-lite - blocks.append(("Spider-2.0 tab", - ["Model", "Spider-2.0-lite"], - ["Inkling", p(spider["accuracy_of_local_135"])])) + blocks.append( + ( + "Spider-2.0 tab", + ["Model", "Spider-2.0-lite"], + ["Inkling", p(spider["accuracy_of_local_135"])], + ) + ) lines = [] for title, header, row in blocks: diff --git a/src/commons_reducto.py b/src/commons_reducto.py index 637af9c..9b45797 100644 --- a/src/commons_reducto.py +++ b/src/commons_reducto.py @@ -2,8 +2,9 @@ import json import os import threading +from collections.abc import Iterable from pathlib import Path -from typing import Any, Iterable +from typing import Any import httpx from dotenv import load_dotenv @@ -18,7 +19,9 @@ ) -reducto_client = Reducto(api_key=REDUCTO_API_KEY) +reducto_client: Any = Reducto( + api_key=REDUCTO_API_KEY +) # SDK stubs are strict; used dynamically # ---------- Usage tracking (thread-safe) ---------------------------------- diff --git a/tests/test_encode_gemini.py b/tests/test_encode_gemini.py index 173264a..48069cc 100644 --- a/tests/test_encode_gemini.py +++ b/tests/test_encode_gemini.py @@ -20,7 +20,12 @@ ) BUDGET_CAPS = Capabilities.from_dict( { - "reasoning": {"style": "thinking_budget", "off_value": 0, "on_value": -1, "true_off": True}, + "reasoning": { + "style": "thinking_budget", + "off_value": 0, + "on_value": -1, + "true_off": True, + }, "media": {"image": "gemini_part"}, "response": "gemini_text", } diff --git a/tests/test_reasoning.py b/tests/test_reasoning.py index 6ec9ff5..8a0b511 100644 --- a/tests/test_reasoning.py +++ b/tests/test_reasoning.py @@ -76,7 +76,10 @@ def test_effort_intermediate_mode_passes_through(): def test_gemini_37_flash_thinking_level_floor_is_low(): cap = ReasoningCap( - style=ReasoningStyle.THINKING_LEVEL, off_value="low", on_value="high", true_off=False + style=ReasoningStyle.THINKING_LEVEL, + off_value="low", + on_value="high", + true_off=False, ) inj = build_reasoning("off", cap) assert inj.thinking_level == "low" @@ -85,7 +88,10 @@ def test_gemini_37_flash_thinking_level_floor_is_low(): def test_gemini_old_flash_thinking_level_floor_is_minimal(): cap = ReasoningCap( - style=ReasoningStyle.THINKING_LEVEL, off_value="minimal", on_value="high", true_off=False + style=ReasoningStyle.THINKING_LEVEL, + off_value="minimal", + on_value="high", + true_off=False, ) assert build_reasoning("off", cap).thinking_level == "minimal" diff --git a/uv.lock b/uv.lock index b3aa3d5..4a92900 100644 --- a/uv.lock +++ b/uv.lock @@ -1265,6 +1265,7 @@ dependencies = [ { name = "pymupdf" }, { name = "python-dotenv" }, { name = "pyyaml" }, + { name = "reducto" }, { name = "ruff" }, { name = "scikit-image" }, { name = "scipy" }, @@ -1318,6 +1319,7 @@ requires-dist = [ { name = "pymupdf", specifier = ">=1.27.2" }, { name = "python-dotenv", specifier = ">=1.2.2" }, { name = "pyyaml", specifier = ">=6.0" }, + { name = "reducto", specifier = ">=0.22.0" }, { name = "ruff", specifier = ">=0.15.5" }, { name = "scikit-image", specifier = ">=0.21.0" }, { name = "scipy", specifier = ">=1.10.0" }, @@ -3243,6 +3245,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/a6/51fc1b0e61e3326e1c68a61cfd0c6b3c34c843681c4b1eefbf0596f59162/rapidfuzz-3.14.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3e91dcd2549b8f8d843f98ba03a17e01f3d8b72ce942adbbb6761bc58ffce813", size = 855409, upload-time = "2026-04-07T11:16:15.787Z" }, ] +[[package]] +name = "reducto" +version = "0.22.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "reductoai" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/84/38/c629dcc4ca3854841b59f692ae4faf2ba58e93a6599413269117253c4c19/reducto-0.22.0.tar.gz", hash = "sha256:02afd1fc733a7869cf59432d114a517cd6a532d0cb10c639db76484380b8fd20", size = 1853, upload-time = "2026-04-17T00:55:26.913Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/98/c2f13eae75d58e38b01157a7c2e0a3109ead4b1c62f9a284a0faf0bb1de2/reducto-0.22.0-py3-none-any.whl", hash = "sha256:b60bee016daffd37d7c3955229ebc4d53df0811280a6c1d37afc2c662db2c9b9", size = 1785, upload-time = "2026-04-17T00:55:25.679Z" }, +] + +[[package]] +name = "reductoai" +version = "0.22.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2d/56/fc608656c68c261a2c61056d5fa8f59440a6a023b8071a5295a10654b4b9/reductoai-0.22.0.tar.gz", hash = "sha256:77965a930627f4f440fe7d7dbe7318c643da589e4c1fefbb2d4106b15af77a71", size = 313933, upload-time = "2026-03-29T01:25:49.705Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/bb/fa663e679011a91f41012cb66aa18e14278ea617868a769369895efd70d0/reductoai-0.22.0-py3-none-any.whl", hash = "sha256:a0c8d9c0372c49d618da83304f62ed0da0aae57fe37abd4072dbe87e206aa2e6", size = 154407, upload-time = "2026-03-29T01:25:48.356Z" }, +] + [[package]] name = "regex" version = "2026.7.19" From b4f0617a53283d17736cb7d648e6c49e231dc490 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Mon, 17 Aug 2026 09:33:50 +0530 Subject: [PATCH 35/38] refactor: move reducto into the core package; rename bench_core -> src src/commons_reducto.py folded into the core (bench_core/), then bench_core/ renamed to src/ so there's a single package. All references updated: imports (from src.X), the CLI entrypoint (python -m src), GitHub Actions (python -m src, ruff check src tests, src/targets.yaml triggers), and the CI scripts. Reducto importers (from src.commons_reducto) resolve to src/commons_reducto.py. Verified: python -m src list-targets, 185 tests, ruff (repo + src/tests), ty check src. --- .github/workflows/add-target.yml | 4 +-- .github/workflows/benchmark.yml | 8 ++--- .github/workflows/tests.yml | 2 +- README.md | 10 +++--- bench_core/providers/__init__.py | 0 benchmarks/asr/bench.py | 2 +- benchmarks/gpqa/bench.py | 2 +- benchmarks/mmmlu/bench.py | 2 +- benchmarks/mmmu_pro/bench.py | 6 ++-- benchmarks/obj_detection/bench.py | 6 ++-- benchmarks/ocrbench_v2/bench.py | 4 +-- benchmarks/olmocr/harness.py | 4 +-- benchmarks/spider2_lite/bench.py | 2 +- scripts/add_target.py | 6 ++-- scripts/ci_matrix.py | 4 +-- scripts/report_scores.py | 4 +-- scripts/run_fireworks_suite.sh | 36 +++++++++---------- {bench_core => src}/__main__.py | 2 +- {bench_core => src}/capabilities.py | 0 {bench_core => src}/cli.py | 6 ++-- {bench_core => src}/config.py | 10 +++--- {bench_core => src}/datautil.py | 0 {bench_core => src}/decode.py | 4 +-- {bench_core => src}/errors.py | 0 {bench_core => src}/execute.py | 8 ++--- {bench_core => src}/media.py | 4 +-- {bench_core => src/providers}/__init__.py | 0 {bench_core => src}/providers/anthropic.py | 10 +++--- {bench_core => src}/providers/base.py | 10 +++--- {bench_core => src}/providers/gemini.py | 10 +++--- .../providers/openai_compat.py | 10 +++--- {bench_core => src}/reasoning.py | 2 +- {bench_core => src}/request.py | 0 {bench_core => src}/response.py | 0 {bench_core => src}/results.py | 0 {bench_core => src}/runner.py | 8 ++--- {bench_core => src}/targets.yaml | 0 tests/test_add_target.py | 4 +-- tests/test_capabilities.py | 2 +- tests/test_config.py | 8 ++--- tests/test_datautil.py | 2 +- tests/test_decode.py | 4 +-- tests/test_encode_anthropic.py | 6 ++-- tests/test_encode_gemini.py | 6 ++-- tests/test_encode_openai_compat.py | 6 ++-- tests/test_errors.py | 2 +- tests/test_execute.py | 12 +++---- tests/test_media.py | 12 +++---- tests/test_reasoning.py | 4 +-- tests/test_results.py | 2 +- tests/test_runner.py | 12 +++---- tests/test_targets.py | 6 ++-- 52 files changed, 137 insertions(+), 137 deletions(-) delete mode 100644 bench_core/providers/__init__.py rename {bench_core => src}/__main__.py (88%) rename {bench_core => src}/capabilities.py (100%) rename {bench_core => src}/cli.py (98%) rename {bench_core => src}/config.py (96%) rename {bench_core => src}/datautil.py (100%) rename {bench_core => src}/decode.py (95%) rename {bench_core => src}/errors.py (100%) rename {bench_core => src}/execute.py (94%) rename {bench_core => src}/media.py (96%) rename {bench_core => src/providers}/__init__.py (100%) rename {bench_core => src}/providers/anthropic.py (91%) rename {bench_core => src}/providers/base.py (82%) rename {bench_core => src}/providers/gemini.py (89%) rename {bench_core => src}/providers/openai_compat.py (88%) rename {bench_core => src}/reasoning.py (97%) rename {bench_core => src}/request.py (100%) rename {bench_core => src}/response.py (100%) rename {bench_core => src}/results.py (100%) rename {bench_core => src}/runner.py (96%) rename {bench_core => src}/targets.yaml (100%) diff --git a/.github/workflows/add-target.yml b/.github/workflows/add-target.yml index 1ead001..d805cdd 100644 --- a/.github/workflows/add-target.yml +++ b/.github/workflows/add-target.yml @@ -54,8 +54,8 @@ jobs: title: "Add benchmark target: ${{ github.event.inputs.name }}" commit-message: "feat(targets): add ${{ github.event.inputs.name }}" body: | - Adds `${{ github.event.inputs.name }}` (${{ github.event.inputs.provider }} / `${{ github.event.inputs.model_id }}`) to `bench_core/targets.yaml`. + Adds `${{ github.event.inputs.name }}` (${{ github.event.inputs.provider }} / `${{ github.event.inputs.model_id }}`) to `src/targets.yaml`. The entry was validated (loads, resolves capabilities, builds its adapter). **Merging this PR triggers the benchmark run for the new target** (benchmark.yml push trigger). - add-paths: bench_core/targets.yaml + add-paths: src/targets.yaml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ebc160f..245ede8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: target: - description: "target name — required (see bench_core/targets.yaml / list-targets)" + description: "target name — required (see src/targets.yaml / list-targets)" required: true benchmarks: description: "comma-separated benchmarks, or 'all' for the CI set" @@ -19,7 +19,7 @@ on: - cron: "0 6 * * 1" push: branches: [main] - paths: [bench_core/targets.yaml] # a merged target change runs a full benchmark for it + paths: [src/targets.yaml] # a merged target change runs a full benchmark for it # never run two paid benchmark passes for the same ref at once (queue, don't cancel # — a half-finished expensive run's spend shouldn't be thrown away) @@ -54,7 +54,7 @@ jobs: INPUT_BENCHMARKS: ${{ github.event.inputs.benchmarks }} run: | if [ "${{ github.event_name }}" = "push" ]; then - git show "${{ github.event.before }}:bench_core/targets.yaml" > /tmp/old_targets.yaml 2>/dev/null || : > /tmp/old_targets.yaml + git show "${{ github.event.before }}:src/targets.yaml" > /tmp/old_targets.yaml 2>/dev/null || : > /tmp/old_targets.yaml export OLD_TARGETS_FILE=/tmp/old_targets.yaml SAMPLE="" # a merged target change gets a full benchmark run else @@ -92,7 +92,7 @@ jobs: run: | SAMPLE="${{ needs.plan.outputs.sample }}" REASONING="${{ needs.plan.outputs.reasoning }}" - uv run python -m bench_core run \ + uv run python -m src run \ --target "${{ matrix.target }}" --benchmark "${{ matrix.benchmark }}" \ ${SAMPLE:+--sample "$SAMPLE"} ${REASONING:+--reasoning "$REASONING"} - name: Upload metrics diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 04f0144..ec3f45d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,6 +23,6 @@ jobs: - name: Install deps run: uv sync --group dev - name: Lint (new code) - run: uv run ruff check bench_core tests + run: uv run ruff check src tests - name: Tests (offline) run: uv run pytest -q diff --git a/README.md b/README.md index 9ed06f0..cc55e50 100644 --- a/README.md +++ b/README.md @@ -23,13 +23,13 @@ FIREWORKS_API_KEY=… ## Run a benchmark -Everything runs through one CLI (`bench_core`). Models are defined in -`bench_core/targets.yaml`; add one there to benchmark a new model — no code. +Everything runs through one CLI (`src`). Models are defined in +`src/targets.yaml`; add one there to benchmark a new model — no code. ```bash -uv run python -m bench_core list-targets # what's available -uv run python -m bench_core run --target inkling --benchmark gpqa # a full run -uv run python -m bench_core run --target gpt-5.5 --benchmark ocrbench_v2 --sample 20 # smoke +uv run python -m src list-targets # what's available +uv run python -m src run --target inkling --benchmark gpqa # a full run +uv run python -m src run --target gpt-5.5 --benchmark ocrbench_v2 --sample 20 # smoke uv run python scripts/report_scores.py # view scores ``` diff --git a/bench_core/providers/__init__.py b/bench_core/providers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/benchmarks/asr/bench.py b/benchmarks/asr/bench.py index 3510f10..83458f5 100644 --- a/benchmarks/asr/bench.py +++ b/benchmarks/asr/bench.py @@ -8,7 +8,7 @@ from jiwer import cer, wer -from bench_core.request import AudioPart, Message, ReasoningSpec, Request, TextPart +from src.request import AudioPart, Message, ReasoningSpec, Request, TextPart NAME = "voxpopuli_aa" ID_KEY = "id" diff --git a/benchmarks/gpqa/bench.py b/benchmarks/gpqa/bench.py index 3c2b1f1..0161e8c 100644 --- a/benchmarks/gpqa/bench.py +++ b/benchmarks/gpqa/bench.py @@ -6,7 +6,7 @@ import re from collections import defaultdict -from bench_core.request import Message, ReasoningSpec, Request, TextPart +from src.request import Message, ReasoningSpec, Request, TextPart NAME = "gpqa" ID_KEY = "id" diff --git a/benchmarks/mmmlu/bench.py b/benchmarks/mmmlu/bench.py index 4127da7..134d8e0 100644 --- a/benchmarks/mmmlu/bench.py +++ b/benchmarks/mmmlu/bench.py @@ -10,7 +10,7 @@ import re from collections import defaultdict -from bench_core.request import Message, ReasoningSpec, Request, TextPart +from src.request import Message, ReasoningSpec, Request, TextPart NAME = "mmmlu" ID_KEY = "id" diff --git a/benchmarks/mmmu_pro/bench.py b/benchmarks/mmmu_pro/bench.py index 0f36d8d..5a120b4 100644 --- a/benchmarks/mmmu_pro/bench.py +++ b/benchmarks/mmmu_pro/bench.py @@ -7,8 +7,8 @@ from collections import defaultdict from typing import Any -from bench_core.media import encode_image -from bench_core.request import Message, ReasoningSpec, Request, TextPart +from src.media import encode_image +from src.request import Message, ReasoningSpec, Request, TextPart NAME = "mmmu_pro" ID_KEY = "id" @@ -103,7 +103,7 @@ def load_samples( global _DATASET if sample_size: - from bench_core.datautil import load_rows + from src.datautil import load_rows rows = load_rows(_DATASET_REPO, _SPLIT, sample_size, config=_CONFIGS[variant]) return [_mk_sample(dict(r), variant) for r in rows] diff --git a/benchmarks/obj_detection/bench.py b/benchmarks/obj_detection/bench.py index a7e31ba..2093e03 100644 --- a/benchmarks/obj_detection/bench.py +++ b/benchmarks/obj_detection/bench.py @@ -11,8 +11,8 @@ from collections import defaultdict from typing import Any -from bench_core.media import encode_image -from bench_core.request import Message, ReasoningSpec, Request, TextPart +from src.media import encode_image +from src.request import Message, ReasoningSpec, Request, TextPart NAME = "refcoco" ID_KEY = "id" @@ -248,7 +248,7 @@ def load_samples(sample_size: int | None = None, variant: str = "val") -> list[d dataset_id, split = _parse_variant(variant) if sample_size: - from bench_core.datautil import load_rows + from src.datautil import load_rows rows = load_rows(dataset_id, split, sample_size) out = (_mk_sample(r, i, r["image"]) for i, r in enumerate(rows)) diff --git a/benchmarks/ocrbench_v2/bench.py b/benchmarks/ocrbench_v2/bench.py index 6bb3ebe..0ac6520 100644 --- a/benchmarks/ocrbench_v2/bench.py +++ b/benchmarks/ocrbench_v2/bench.py @@ -15,8 +15,8 @@ from pathlib import Path from typing import Any -from bench_core.media import encode_image -from bench_core.request import Message, ReasoningSpec, Request, TextPart +from src.media import encode_image +from src.request import Message, ReasoningSpec, Request, TextPart NAME = "ocrbench_v2" ID_KEY = "id" diff --git a/benchmarks/olmocr/harness.py b/benchmarks/olmocr/harness.py index 488e70a..d787234 100644 --- a/benchmarks/olmocr/harness.py +++ b/benchmarks/olmocr/harness.py @@ -14,8 +14,8 @@ import sys from pathlib import Path -from bench_core.request import ImagePart, Message, ReasoningSpec, Request, TextPart -from bench_core.results import model_slug +from src.request import ImagePart, Message, ReasoningSpec, Request, TextPart +from src.results import model_slug NAME = "olmocr" ID_KEY = "id" diff --git a/benchmarks/spider2_lite/bench.py b/benchmarks/spider2_lite/bench.py index b704d9d..b727c73 100644 --- a/benchmarks/spider2_lite/bench.py +++ b/benchmarks/spider2_lite/bench.py @@ -13,7 +13,7 @@ import time from pathlib import Path -from bench_core.request import Message, ReasoningSpec, Request, TextPart +from src.request import Message, ReasoningSpec, Request, TextPart NAME = "spider2_lite" ID_KEY = "instance_id" diff --git a/scripts/add_target.py b/scripts/add_target.py index 206268f..b799cbd 100644 --- a/scripts/add_target.py +++ b/scripts/add_target.py @@ -1,4 +1,4 @@ -"""Append a target to bench_core/targets.yaml from UI/CLI inputs, then validate. +"""Append a target to src/targets.yaml from UI/CLI inputs, then validate. Used by the add-target GitHub workflow (which opens a PR) and locally: @@ -25,14 +25,14 @@ ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) -from bench_core.config import ( +from src.config import ( PROVIDERS, build_adapter, load_all_targets, resolve_capabilities, ) -TARGETS_FILE = ROOT / "bench_core" / "targets.yaml" +TARGETS_FILE = ROOT / "src" / "targets.yaml" def _entry_text(name, provider, model_id, capabilities, ci_regression) -> str: diff --git a/scripts/ci_matrix.py b/scripts/ci_matrix.py index d13bde2..27b4e9c 100644 --- a/scripts/ci_matrix.py +++ b/scripts/ci_matrix.py @@ -61,7 +61,7 @@ def build_matrix( targets: list[str] | None = None, ) -> dict: if event == "schedule": - from bench_core.config import load_all_targets + from src.config import load_all_targets flagged = [ n for n, t in load_all_targets().items() if t.raw.get("ci_regression") @@ -86,7 +86,7 @@ def build_matrix( if event == "push": tl = changed_targets( os.getenv("OLD_TARGETS_FILE"), - os.getenv("NEW_TARGETS_FILE", "bench_core/targets.yaml"), + os.getenv("NEW_TARGETS_FILE", "src/targets.yaml"), ) print(json.dumps(build_matrix("push", targets=tl))) else: diff --git a/scripts/report_scores.py b/scripts/report_scores.py index 6bb0e2b..0049224 100644 --- a/scripts/report_scores.py +++ b/scripts/report_scores.py @@ -21,7 +21,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(ROOT)) # so `bench_core` is importable when run as a script +sys.path.insert(0, str(ROOT)) # so `src` is importable when run as a script RESULTS = ROOT / "results" LOGS = ROOT / "logs" @@ -171,7 +171,7 @@ def _pct(x): def load_metrics() -> dict: """{model: {benchmark_key: payload}} from the results/// metrics.json contract (written by the CLI).""" - from bench_core.results import discover + from src.results import discover out: dict[str, dict] = {} for d in discover(RESULTS): diff --git a/scripts/run_fireworks_suite.sh b/scripts/run_fireworks_suite.sh index 9720250..25bf882 100755 --- a/scripts/run_fireworks_suite.sh +++ b/scripts/run_fireworks_suite.sh @@ -68,61 +68,61 @@ run_step() { # after ~233s per sample (every sample would burn 3 retries and fail), while # effort=none answers in ~1.3s. Transcription gains nothing from thinking. if [ -n "${TARGET:-}" ]; then - run_step asr uv run python -m bench_core run --benchmark asr --target "$TARGET" + run_step asr uv run python -m src run --benchmark asr --target "$TARGET" else - echo " [skip] asr: set TARGET= (python -m bench_core list-targets)" + echo " [skip] asr: set TARGET= (python -m src list-targets)" fi if [ -n "${TARGET:-}" ]; then - run_step spider2 uv run python -m bench_core run --benchmark spider2 --target "$TARGET" + run_step spider2 uv run python -m src run --benchmark spider2 --target "$TARGET" else - echo " [skip] spider2: set TARGET= (python -m bench_core list-targets)" + echo " [skip] spider2: set TARGET= (python -m src list-targets)" fi if [ -n "${TARGET:-}" ]; then R="$([ "$REASONING" = "high" ] && echo high || echo off)" - run_step mmmupro_standard uv run python -m bench_core run --benchmark mmmu_pro \ + run_step mmmupro_standard uv run python -m src run --benchmark mmmu_pro \ --target "$TARGET" --variant standard --reasoning "$R" - run_step mmmupro_vision uv run python -m bench_core run --benchmark mmmu_pro \ + run_step mmmupro_vision uv run python -m src run --benchmark mmmu_pro \ --target "$TARGET" --variant vision --reasoning "$R" else - echo " [skip] mmmupro: set TARGET= (python -m bench_core list-targets)" + echo " [skip] mmmupro: set TARGET= (python -m src list-targets)" fi if [ -n "${TARGET:-}" ]; then - run_step refcoco uv run python -m bench_core run --benchmark refcoco \ + run_step refcoco uv run python -m src run --benchmark refcoco \ --target "$TARGET" --variant "$REFCOCO_SPLIT" else - echo " [skip] refcoco: set TARGET= (python -m bench_core list-targets)" + echo " [skip] refcoco: set TARGET= (python -m src list-targets)" fi if [ -n "${TARGET:-}" ]; then - run_step olmocr uv run python -m bench_core run --benchmark olmocr --target "$TARGET" + run_step olmocr uv run python -m src run --benchmark olmocr --target "$TARGET" else - echo " [skip] olmocr: set TARGET= (python -m bench_core list-targets)" + echo " [skip] olmocr: set TARGET= (python -m src list-targets)" fi if [ -n "${TARGET:-}" ]; then - run_step mmmlu uv run python -m bench_core run --benchmark mmmlu --target "$TARGET" \ + run_step mmmlu uv run python -m src run --benchmark mmmlu --target "$TARGET" \ --reasoning "$([ "$REASONING" = "high" ] && echo high || echo off)" else - echo " [skip] mmmlu: set TARGET= (python -m bench_core list-targets)" + echo " [skip] mmmlu: set TARGET= (python -m src list-targets)" fi if [ -n "${TARGET:-}" ]; then - run_step ocrbench uv run python -m bench_core run --benchmark ocrbench_v2 --target "$TARGET" + run_step ocrbench uv run python -m src run --benchmark ocrbench_v2 --target "$TARGET" else - echo " [skip] ocrbench: set TARGET= (python -m bench_core list-targets)" + echo " [skip] ocrbench: set TARGET= (python -m src list-targets)" fi # Gated dataset — needs `hf auth login` plus accepted terms on the Hub. # GPQA now runs through the unified CLI; provider/model/reasoning come from the -# target (bench_core/targets.yaml), so set TARGET= (see `list-targets`). +# target (src/targets.yaml), so set TARGET= (see `list-targets`). if [ -n "${TARGET:-}" ]; then - run_step gpqa uv run python -m bench_core run --benchmark gpqa --target "$TARGET" \ + run_step gpqa uv run python -m src run --benchmark gpqa --target "$TARGET" \ --reasoning "$([ "$REASONING" = "high" ] && echo high || echo off)" else - echo " [skip] gpqa: set TARGET= (python -m bench_core list-targets)" + echo " [skip] gpqa: set TARGET= (python -m src list-targets)" fi # The format-tolerant oracle (upper bound) is now computed inline by the refcoco diff --git a/bench_core/__main__.py b/src/__main__.py similarity index 88% rename from bench_core/__main__.py rename to src/__main__.py index bd0698a..64c79f7 100644 --- a/bench_core/__main__.py +++ b/src/__main__.py @@ -4,7 +4,7 @@ import os import sys -from bench_core.cli import main +from src.cli import main main() sys.stdout.flush() diff --git a/bench_core/capabilities.py b/src/capabilities.py similarity index 100% rename from bench_core/capabilities.py rename to src/capabilities.py diff --git a/bench_core/cli.py b/src/cli.py similarity index 98% rename from bench_core/cli.py rename to src/cli.py index e8c5198..b3fe5d3 100644 --- a/bench_core/cli.py +++ b/src/cli.py @@ -5,13 +5,13 @@ import importlib import inspect -from bench_core.config import ( +from src.config import ( build_routes, load_all_targets, load_target, ) -from bench_core.results import RunStore -from bench_core.runner import run_benchmark +from src.results import RunStore +from src.runner import run_benchmark BENCHMARKS = { "gpqa": "benchmarks.gpqa.bench", diff --git a/bench_core/config.py b/src/config.py similarity index 96% rename from bench_core/config.py rename to src/config.py index e928041..346c4a5 100644 --- a/bench_core/config.py +++ b/src/config.py @@ -7,10 +7,10 @@ import yaml -from bench_core.capabilities import Capabilities, deep_merge -from bench_core.providers.anthropic import AnthropicAdapter -from bench_core.providers.gemini import GeminiAdapter -from bench_core.providers.openai_compat import OpenAICompatAdapter +from src.capabilities import Capabilities, deep_merge +from src.providers.anthropic import AnthropicAdapter +from src.providers.gemini import GeminiAdapter +from src.providers.openai_compat import OpenAICompatAdapter @dataclass @@ -220,7 +220,7 @@ def build_adapter(target: Target): def build_routes( target: Target, benchmark: str | None = None, cli_overrides: dict | None = None ): - from bench_core.runner import Route + from src.runner import Route specs = [target] + [ Target( diff --git a/bench_core/datautil.py b/src/datautil.py similarity index 100% rename from bench_core/datautil.py rename to src/datautil.py diff --git a/bench_core/decode.py b/src/decode.py similarity index 95% rename from bench_core/decode.py rename to src/decode.py index 2b44568..f57a6a2 100644 --- a/bench_core/decode.py +++ b/src/decode.py @@ -4,8 +4,8 @@ from typing import Any -from bench_core.capabilities import ResponseShape -from bench_core.response import Response +from src.capabilities import ResponseShape +from src.response import Response def _get(obj: Any, name: str, default=None): diff --git a/bench_core/errors.py b/src/errors.py similarity index 100% rename from bench_core/errors.py rename to src/errors.py diff --git a/bench_core/execute.py b/src/execute.py similarity index 94% rename from bench_core/execute.py rename to src/execute.py index de58ac5..8421111 100644 --- a/bench_core/execute.py +++ b/src/execute.py @@ -4,10 +4,10 @@ import dataclasses -from bench_core.capabilities import Capabilities, ReasoningStyle -from bench_core.errors import Classification, ErrorKind -from bench_core.request import Request -from bench_core.response import Response +from src.capabilities import Capabilities, ReasoningStyle +from src.errors import Classification, ErrorKind +from src.request import Request +from src.response import Response _THINKING_FLOORS = ["minimal", "low", "medium", "high"] diff --git a/bench_core/media.py b/src/media.py similarity index 96% rename from bench_core/media.py rename to src/media.py index f5af87d..99bc7be 100644 --- a/bench_core/media.py +++ b/src/media.py @@ -7,8 +7,8 @@ from dataclasses import dataclass from typing import Any -from bench_core.capabilities import AudioShape, ImageShape -from bench_core.request import AudioPart, ImagePart +from src.capabilities import AudioShape, ImageShape +from src.request import AudioPart, ImagePart #Marker for a native `google.genai` part diff --git a/bench_core/__init__.py b/src/providers/__init__.py similarity index 100% rename from bench_core/__init__.py rename to src/providers/__init__.py diff --git a/bench_core/providers/anthropic.py b/src/providers/anthropic.py similarity index 91% rename from bench_core/providers/anthropic.py rename to src/providers/anthropic.py index b1573d1..80c5ea1 100644 --- a/bench_core/providers/anthropic.py +++ b/src/providers/anthropic.py @@ -1,10 +1,10 @@ from __future__ import annotations -from bench_core.capabilities import Capabilities -from bench_core.media import image_block -from bench_core.providers.base import ProviderAdapter -from bench_core.reasoning import build_reasoning -from bench_core.request import ImagePart, Message, Request, TextPart +from src.capabilities import Capabilities +from src.media import image_block +from src.providers.base import ProviderAdapter +from src.reasoning import build_reasoning +from src.request import ImagePart, Message, Request, TextPart _DEFAULT_OFF_MAX = 1024 _DEFAULT_ON_MAX = 16000 diff --git a/bench_core/providers/base.py b/src/providers/base.py similarity index 82% rename from bench_core/providers/base.py rename to src/providers/base.py index 0952a0d..e1e7df8 100644 --- a/bench_core/providers/base.py +++ b/src/providers/base.py @@ -3,11 +3,11 @@ import os from typing import Any -from bench_core.capabilities import Capabilities -from bench_core.decode import decode_response -from bench_core.errors import Classification, classify -from bench_core.request import Request -from bench_core.response import Response +from src.capabilities import Capabilities +from src.decode import decode_response +from src.errors import Classification, classify +from src.request import Request +from src.response import Response class ProviderAdapter: diff --git a/bench_core/providers/gemini.py b/src/providers/gemini.py similarity index 89% rename from bench_core/providers/gemini.py rename to src/providers/gemini.py index 50b74e6..f86c0c8 100644 --- a/bench_core/providers/gemini.py +++ b/src/providers/gemini.py @@ -3,11 +3,11 @@ from __future__ import annotations -from bench_core.capabilities import Capabilities -from bench_core.media import GeminiPart, audio_block, image_block -from bench_core.providers.base import ProviderAdapter -from bench_core.reasoning import build_reasoning -from bench_core.request import AudioPart, ImagePart, Request, TextPart +from src.capabilities import Capabilities +from src.media import GeminiPart, audio_block, image_block +from src.providers.base import ProviderAdapter +from src.reasoning import build_reasoning +from src.request import AudioPart, ImagePart, Request, TextPart class GeminiAdapter(ProviderAdapter): diff --git a/bench_core/providers/openai_compat.py b/src/providers/openai_compat.py similarity index 88% rename from bench_core/providers/openai_compat.py rename to src/providers/openai_compat.py index 029d276..fee87e6 100644 --- a/bench_core/providers/openai_compat.py +++ b/src/providers/openai_compat.py @@ -4,11 +4,11 @@ from typing import Any -from bench_core.capabilities import Capabilities -from bench_core.media import audio_block, image_block -from bench_core.providers.base import ProviderAdapter -from bench_core.reasoning import build_reasoning -from bench_core.request import AudioPart, ImagePart, Message, Request, TextPart +from src.capabilities import Capabilities +from src.media import audio_block, image_block +from src.providers.base import ProviderAdapter +from src.reasoning import build_reasoning +from src.request import AudioPart, ImagePart, Message, Request, TextPart class OpenAICompatAdapter(ProviderAdapter): diff --git a/bench_core/reasoning.py b/src/reasoning.py similarity index 97% rename from bench_core/reasoning.py rename to src/reasoning.py index 720e09d..f88123e 100644 --- a/bench_core/reasoning.py +++ b/src/reasoning.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field from typing import Any, assert_never -from bench_core.capabilities import ReasoningCap, ReasoningStyle +from src.capabilities import ReasoningCap, ReasoningStyle _HIGH_MODES = {"high", "on", "max"} _OFF_MODES = {"off", "none", "disabled"} diff --git a/bench_core/request.py b/src/request.py similarity index 100% rename from bench_core/request.py rename to src/request.py diff --git a/bench_core/response.py b/src/response.py similarity index 100% rename from bench_core/response.py rename to src/response.py diff --git a/bench_core/results.py b/src/results.py similarity index 100% rename from bench_core/results.py rename to src/results.py diff --git a/bench_core/runner.py b/src/runner.py similarity index 96% rename from bench_core/runner.py rename to src/runner.py index cb80146..e1af7a6 100644 --- a/bench_core/runner.py +++ b/src/runner.py @@ -6,10 +6,10 @@ import asyncio from dataclasses import dataclass, field -from bench_core.capabilities import Capabilities -from bench_core.errors import ErrorKind -from bench_core.execute import BenchError, execute -from bench_core.results import RunStore +from src.capabilities import Capabilities +from src.errors import ErrorKind +from src.execute import BenchError, execute +from src.results import RunStore _RETRYABLE = {ErrorKind.RATE_LIMITED, ErrorKind.TRANSIENT, ErrorKind.EMPTY_CONTENT} # Advance to the next provider route only when the host itself won't serve. diff --git a/bench_core/targets.yaml b/src/targets.yaml similarity index 100% rename from bench_core/targets.yaml rename to src/targets.yaml diff --git a/tests/test_add_target.py b/tests/test_add_target.py index ceb2ba3..2b3bf10 100644 --- a/tests/test_add_target.py +++ b/tests/test_add_target.py @@ -12,8 +12,8 @@ from add_target import add_target from ci_matrix import changed_targets -from bench_core.capabilities import ReasoningStyle -from bench_core.config import load_all_targets, resolve_capabilities +from src.capabilities import ReasoningStyle +from src.config import load_all_targets, resolve_capabilities def _base(tmp_path): diff --git a/tests/test_capabilities.py b/tests/test_capabilities.py index 7ae4d62..a2c822c 100644 --- a/tests/test_capabilities.py +++ b/tests/test_capabilities.py @@ -6,7 +6,7 @@ gives the adapters a typed, validated view. """ -from bench_core.capabilities import ( +from src.capabilities import ( AudioShape, Capabilities, ImageShape, diff --git a/tests/test_config.py b/tests/test_config.py index 27a63a0..db58421 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -8,16 +8,16 @@ import pytest -from bench_core.capabilities import AudioShape, ReasoningStyle -from bench_core.config import ( +from src.capabilities import AudioShape, ReasoningStyle +from src.config import ( build_adapter, build_routes, load_all_targets, load_target, resolve_capabilities, ) -from bench_core.providers.gemini import GeminiAdapter -from bench_core.providers.openai_compat import OpenAICompatAdapter +from src.providers.gemini import GeminiAdapter +from src.providers.openai_compat import OpenAICompatAdapter def _targets(tmp_path, body): diff --git a/tests/test_datautil.py b/tests/test_datautil.py index 55c56d5..04b8af3 100644 --- a/tests/test_datautil.py +++ b/tests/test_datautil.py @@ -1,7 +1,7 @@ import sys import types -from bench_core.datautil import load_rows +from src.datautil import load_rows def _install_fake(monkeypatch, capture): diff --git a/tests/test_decode.py b/tests/test_decode.py index 15d2512..2161913 100644 --- a/tests/test_decode.py +++ b/tests/test_decode.py @@ -6,8 +6,8 @@ from types import SimpleNamespace as NS -from bench_core.capabilities import ResponseShape -from bench_core.decode import decode_response +from src.capabilities import ResponseShape +from src.decode import decode_response def _openai_raw( diff --git a/tests/test_encode_anthropic.py b/tests/test_encode_anthropic.py index f166b6b..b755085 100644 --- a/tests/test_encode_anthropic.py +++ b/tests/test_encode_anthropic.py @@ -3,9 +3,9 @@ coupling (all declarative via the cap). """ -from bench_core.capabilities import Capabilities -from bench_core.providers.anthropic import AnthropicAdapter -from bench_core.request import ImagePart, Message, ReasoningSpec, Request, TextPart +from src.capabilities import Capabilities +from src.providers.anthropic import AnthropicAdapter +from src.request import ImagePart, Message, ReasoningSpec, Request, TextPart CAPS = Capabilities.from_dict( { diff --git a/tests/test_encode_gemini.py b/tests/test_encode_gemini.py index 48069cc..8439b36 100644 --- a/tests/test_encode_gemini.py +++ b/tests/test_encode_gemini.py @@ -2,9 +2,9 @@ field (level vs budget) and that media parts become genai Parts. Uses the real google-genai types (offline — no network).""" -from bench_core.capabilities import Capabilities -from bench_core.providers.gemini import GeminiAdapter -from bench_core.request import ImagePart, Message, ReasoningSpec, Request, TextPart +from src.capabilities import Capabilities +from src.providers.gemini import GeminiAdapter +from src.request import ImagePart, Message, ReasoningSpec, Request, TextPart LEVEL_CAPS = Capabilities.from_dict( { diff --git a/tests/test_encode_openai_compat.py b/tests/test_encode_openai_compat.py index a757f37..29eb9db 100644 --- a/tests/test_encode_openai_compat.py +++ b/tests/test_encode_openai_compat.py @@ -3,9 +3,9 @@ together for openai / fireworks / openrouter / interfaze. """ -from bench_core.capabilities import Capabilities -from bench_core.providers.openai_compat import OpenAICompatAdapter -from bench_core.request import ImagePart, Message, ReasoningSpec, Request, TextPart +from src.capabilities import Capabilities +from src.providers.openai_compat import OpenAICompatAdapter +from src.request import ImagePart, Message, ReasoningSpec, Request, TextPart OPENAI_CAPS = Capabilities.from_dict( { diff --git a/tests/test_errors.py b/tests/test_errors.py index 025bc2d..278d7b6 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -4,7 +4,7 @@ mandatory", GPT-5 temperature-under-reasoning). """ -from bench_core.errors import ErrorKind, classify +from src.errors import ErrorKind, classify class FakeAPIError(Exception): diff --git a/tests/test_execute.py b/tests/test_execute.py index 65107c4..00a3566 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -6,11 +6,11 @@ import pytest -from bench_core.capabilities import Capabilities -from bench_core.errors import ErrorKind -from bench_core.execute import BenchError, execute -from bench_core.request import Message, ReasoningSpec, Request, TextPart -from bench_core.response import Response +from src.capabilities import Capabilities +from src.errors import ErrorKind +from src.execute import BenchError, execute +from src.request import Message, ReasoningSpec, Request, TextPart +from src.response import Response class FakeAPIError(Exception): @@ -47,7 +47,7 @@ def decode(self, raw, caps): return Response(text=self.final_text) def classify_error(self, exc): - from bench_core.errors import classify + from src.errors import classify return classify(exc) diff --git a/tests/test_media.py b/tests/test_media.py index 9429ac7..5aa24de 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -8,9 +8,9 @@ import base64 import io -from bench_core.capabilities import AudioShape, ImageShape -from bench_core.media import GeminiPart, audio_block, image_block -from bench_core.request import AudioPart, ImagePart +from src.capabilities import AudioShape, ImageShape +from src.media import GeminiPart, audio_block, image_block +from src.request import AudioPart, ImagePart WAV = b"RIFF....WAVEfake-audio-bytes" JPEG = b"\xff\xd8\xff\xe0jpeg-bytes" @@ -90,7 +90,7 @@ def test_encode_image_converts_rgba_to_jpeg_without_crashing(): # transparent PNG sample doesn't crash the run (it did in 8/9 OCR variants). from PIL import Image - from bench_core.media import encode_image + from src.media import encode_image rgba = Image.new("RGBA", (10, 10), (255, 0, 0, 128)) part = encode_image(rgba, mime="image/jpeg") @@ -101,7 +101,7 @@ def test_encode_image_converts_rgba_to_jpeg_without_crashing(): def test_encode_image_downscales_to_max_side(): from PIL import Image - from bench_core.media import encode_image + from src.media import encode_image big = Image.new("RGB", (3000, 1500)) part = encode_image(big, mime="image/jpeg", max_side=1536) @@ -111,7 +111,7 @@ def test_encode_image_downscales_to_max_side(): def test_encode_image_png_keeps_alpha(): from PIL import Image - from bench_core.media import encode_image + from src.media import encode_image rgba = Image.new("RGBA", (10, 10), (0, 255, 0, 64)) part = encode_image(rgba, mime="image/png") diff --git a/tests/test_reasoning.py b/tests/test_reasoning.py index 8a0b511..dff175b 100644 --- a/tests/test_reasoning.py +++ b/tests/test_reasoning.py @@ -12,8 +12,8 @@ - Moonshot: reasoning.enabled=false + provider pin """ -from bench_core.capabilities import ReasoningCap, ReasoningStyle -from bench_core.reasoning import build_reasoning +from src.capabilities import ReasoningCap, ReasoningStyle +from src.reasoning import build_reasoning def test_none_style_injects_nothing(): diff --git a/tests/test_results.py b/tests/test_results.py index 9fb124b..d395aeb 100644 --- a/tests/test_results.py +++ b/tests/test_results.py @@ -6,7 +6,7 @@ import json -from bench_core.results import RunStore, discover, model_slug +from src.results import RunStore, discover, model_slug def test_sampled_run_is_isolated_from_full_run(tmp_path): diff --git a/tests/test_runner.py b/tests/test_runner.py index 4e05273..87d986f 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -4,11 +4,11 @@ (backoff_base=0). """ -from bench_core.capabilities import Capabilities -from bench_core.request import Message, ReasoningSpec, Request, TextPart -from bench_core.response import Response -from bench_core.results import RunStore -from bench_core.runner import Route, run_benchmark +from src.capabilities import Capabilities +from src.request import Message, ReasoningSpec, Request, TextPart +from src.response import Response +from src.results import RunStore +from src.runner import Route, run_benchmark def _route(adapter, provider="test", caps=None): @@ -46,7 +46,7 @@ def decode(self, raw, caps): return Response(text=raw["text"]) def classify_error(self, exc): - from bench_core.errors import classify + from src.errors import classify return classify(exc) diff --git a/tests/test_targets.py b/tests/test_targets.py index 2e88dea..9b60fbd 100644 --- a/tests/test_targets.py +++ b/tests/test_targets.py @@ -6,15 +6,15 @@ import pytest -from bench_core.capabilities import Capabilities -from bench_core.config import build_adapter, load_all_targets, resolve_capabilities +from src.capabilities import Capabilities +from src.config import build_adapter, load_all_targets, resolve_capabilities TARGETS = load_all_targets() TARGET_NAMES = sorted(TARGETS) def test_targets_exist(): - assert TARGET_NAMES, "no targets defined in bench_core/targets.yaml" + assert TARGET_NAMES, "no targets defined in src/targets.yaml" @pytest.mark.parametrize("name", TARGET_NAMES) From 3e5cd2f54ac81e5232ad11d53ce6f38dcf2a7c6c Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Mon, 17 Aug 2026 09:38:22 +0530 Subject: [PATCH 36/38] refactor: integrate reducto as src/providers/reducto.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reducto was left as a stray src/commons_reducto.py — the old commons_* shape the rest of the refactor eliminated. Moved it into the providers package alongside the other provider integrations (anthropic/gemini/openai_compat); importers now use from src.providers.reducto. Also polished the README CLI reference to python -m src. Verified: ruff (repo + src/tests), ty check src, 185 tests, python -m src. --- README.md | 2 +- benchmarks/olmocr/bench/runners/run_reducto.py | 2 +- benchmarks/olmocr/olmocr_bench_reducto.py | 2 +- src/{commons_reducto.py => providers/reducto.py} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename src/{commons_reducto.py => providers/reducto.py} (100%) diff --git a/README.md b/README.md index cc55e50..dda5ff1 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ FIREWORKS_API_KEY=… ## Run a benchmark -Everything runs through one CLI (`src`). Models are defined in +Everything runs through one CLI (`python -m src`). Models are defined in `src/targets.yaml`; add one there to benchmark a new model — no code. ```bash diff --git a/benchmarks/olmocr/bench/runners/run_reducto.py b/benchmarks/olmocr/bench/runners/run_reducto.py index df1db55..f663ace 100644 --- a/benchmarks/olmocr/bench/runners/run_reducto.py +++ b/benchmarks/olmocr/bench/runners/run_reducto.py @@ -84,7 +84,7 @@ def run_reducto( timeout=timeout, ) try: - from src.commons_reducto import record_usage + from src.providers.reducto import record_usage record_usage("parse", getattr(response, "usage", None)) except Exception: diff --git a/benchmarks/olmocr/olmocr_bench_reducto.py b/benchmarks/olmocr/olmocr_bench_reducto.py index 82a4a28..039e25b 100644 --- a/benchmarks/olmocr/olmocr_bench_reducto.py +++ b/benchmarks/olmocr/olmocr_bench_reducto.py @@ -177,7 +177,7 @@ async def generate_outputs(data_dir: Path): print(f"Done: {num_success} succeeded, {num_failed} failed") try: - from src.commons_reducto import write_usage_snapshot + from src.providers.reducto import write_usage_snapshot write_usage_snapshot(USAGE_OUTPUT) print(f"Usage written to {USAGE_OUTPUT}") diff --git a/src/commons_reducto.py b/src/providers/reducto.py similarity index 100% rename from src/commons_reducto.py rename to src/providers/reducto.py From 7967e74675d2198daa85e48994810e2de14fde7d Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Mon, 17 Aug 2026 09:55:16 +0530 Subject: [PATCH 37/38] feat(targets): restore gemini-2.5-pro, gemini-3.1-pro-preview, gpt-5.4-mini These were per-runner defaults in main (refcoco_gemini, gpqa_gemini/ocrbench pro, openai mini) but weren't carried into targets.yaml. Added for full model parity with main. No ci_regression flag -> available for manual dispatch, not auto-run. gemini-2.5-pro uses the 2.5 thinking_budget knob; the others use provider defaults. --- src/targets.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/targets.yaml b/src/targets.yaml index 6cdc029..df60723 100644 --- a/src/targets.yaml +++ b/src/targets.yaml @@ -51,11 +51,25 @@ targets: capabilities: reasoning: { style: thinking_budget, off_value: 0, on_value: -1, true_off: true } + gemini-2.5-pro: + provider: gemini + model_id: gemini-2.5-pro + capabilities: + reasoning: { style: thinking_budget, off_value: 0, on_value: -1, true_off: true } + + gemini-3.1-pro-preview: + provider: gemini + model_id: gemini-3.1-pro-preview + # --- OpenAI / Anthropic / Interfaze: provider defaults fit as-is --- gpt-5.5: provider: openai model_id: gpt-5.5 + gpt-5.4-mini: + provider: openai + model_id: gpt-5.4-mini + claude-sonnet-4-6: provider: anthropic model_id: claude-sonnet-4-6 From 3990ca5ab546a77a9a5d5d8324de09b7b01faea0 Mon Sep 17 00:00:00 2001 From: Abhinavexist Date: Mon, 17 Aug 2026 17:11:31 +0530 Subject: [PATCH 38/38] chore: remove scripts (temp use) --- scripts/plot_gemini_vs_deepseek.py | 277 --------------------- scripts/plot_inkling_radar.py | 379 ----------------------------- scripts/run_fireworks_suite.sh | 131 ---------- scripts/sheet_columns.py | 303 ----------------------- 4 files changed, 1090 deletions(-) delete mode 100644 scripts/plot_gemini_vs_deepseek.py delete mode 100644 scripts/plot_inkling_radar.py delete mode 100755 scripts/run_fireworks_suite.sh delete mode 100644 scripts/sheet_columns.py diff --git a/scripts/plot_gemini_vs_deepseek.py b/scripts/plot_gemini_vs_deepseek.py deleted file mode 100644 index 6a95c20..0000000 --- a/scripts/plot_gemini_vs_deepseek.py +++ /dev/null @@ -1,277 +0,0 @@ -""" -Gemini-3.5-flash vs DeepSeek-V4-Flash across the benchmark suite. - -Bars, not a radar: DeepSeek V4 Flash is text-only, so 6 of 9 benchmarks are not -runnable for it at all. On a radar those axes would collapse to the centre and -read as "scored badly"; as bars, an absent bar can be labelled for what it is -("no image input") and never be mistaken for a zero. - -Gemini figures come from the CSVs in the repo root (the team's own runs). -DeepSeek figures are measured by this repo. - - uv run python scripts/plot_gemini_vs_deepseek.py -""" - -import csv -import json -from pathlib import Path - -import matplotlib - -matplotlib.use("Agg") -import matplotlib.pyplot as plt - -ROOT = Path(__file__).resolve().parent.parent -RESULTS = ROOT / "results" -CSV_MASTER = ROOT / "Interfaze Benchmarks - running.csv" -OUT = RESULTS / "gemini35flash_vs_deepseekv4flash.png" - -# Validated palette (dataviz references/palette.md), light mode. -SURFACE = "#fcfcfb" -GEMINI = "#2a78d6" # slot 1 -DEEPSEEK = "#eb6834" # slot 2 -INK = "#0b0b0b" -INK_MUTED = "#898781" -GRID = "#e1e0d9" -MONO = ["DejaVu Sans Mono", "Menlo", "monospace"] - -GEMINI_COL = "Gemini-3.5-flash" - -# Why DeepSeek has no number, verified by direct probe against the Fireworks -# endpoint rather than assumed from docs: -# image -> HTTP 400 "This model does not support image inputs" -# audio -> HTTP 400, content must be a plain string -UNSUPPORTED = { - "OCRBench V2": "no image input", - "olmOCR": "no image input", - "RefCoco": "no image input", - "MMMU-Pro": "no image input", - "VoxPoppuliCleaned-AA": "no audio input", - "TheSOB": "not in this repo", -} - -# Benchmark label -> our measured metrics file + extractor. -# NB: gpqa_fireworks' model_slug strips hyphens (deepseekv4flash) while the -# other runners keep them (deepseek-v4-flash) — inherited from gpqa_openrouter. -MEASURED = { - "Spider-2.0-lite": ( - "spider2_lite_local_fireworks_deepseek-v4-flash_metrics.json", - lambda m: m["accuracy_of_local_135"] * 100, - ), - "MMMLU": ( - "mmmlulite_fireworks_accounts-fireworks-models-deepseek-v4-flash_reasoningoff_metrics.json", - lambda m: m["macro_accuracy"] * 100, - ), - "GPQA-Diamond": ( - "fireworks_deepseekv4flash_thinkingoff_gpqa_diamond_metrics.json", - lambda m: m["accuracy"] * 100, - ), -} - -DISPLAY = { - "OCRBench V2": "OCRBench V2", - "olmOCR": "olmOCR", - "RefCoco": "RefCOCO", - "VoxPoppuliCleaned-AA": "VoxPopuli-AA", - "TheSOB": "TheSOB", - "Spider-2.0-lite": "Spider-2.0-Lite", - "GPQA-Diamond": "GPQA Diamond", - "MMMLU": "MMMLU", - "MMMU-Pro": "MMMU-Pro", -} - - -def load_gemini() -> dict: - """Pull the Gemini-3.5-flash column out of the team's master CSV.""" - out = {} - with open(CSV_MASTER, newline="", encoding="utf-8") as f: - for row in csv.DictReader(f): - bench = (row.get("Benchmark") or "").strip() - raw = (row.get(GEMINI_COL) or "").strip() - if not bench or not raw: - continue - try: - out[bench] = float(raw) - except ValueError: - continue # price rows, "Does not support ..." cells - return out - - -# A run in flight is its own state. Its partial accuracy is NOT usable as an -# estimate: mmmlu_multi walks languages in order, so a half-finished run covers -# ~7 of 14 languages and its macro average is biased by construction. -IN_FLIGHT = { - "MMMLU": ( - "mmmlulite_fireworks_accounts-fireworks-models-deepseek-v4-flash_reasoningoff_responses.jsonl", - 19950, - ), - "GPQA-Diamond": ( - "fireworks_deepseekv4flash_thinkingoff_gpqa_diamond_responses.jsonl", - 198, - ), -} - - -def load_deepseek() -> dict: - out = {} - for bench, (fname, pick) in MEASURED.items(): - p = RESULTS / fname - if p.exists(): - out[bench] = round(pick(json.loads(p.read_text())), 2) - return out - - -def pending_note(bench: str) -> str: - """'running (n/total)' for an unfinished run, else a plain not-run.""" - spec = IN_FLIGHT.get(bench) - if not spec: - return UNSUPPORTED.get(bench, "not run") - fname, total = spec - p = RESULTS / fname - done = sum(1 for _ in p.open()) if p.exists() else 0 - return f"running… {done:,}/{total:,}" - - -def main(): - gem = load_gemini() - ds = load_deepseek() - - # Comparable rows first — that is the actual comparison, so it reads first. - comparable = [b for b in DISPLAY if b in ds and b in gem] - others = [b for b in DISPLAY if b not in ds and b in gem] - order = comparable + others - - fig, ax = plt.subplots(figsize=(12.5, 7.6), facecolor=SURFACE) - ax.set_facecolor(SURFACE) - - h = 0.34 - gap = 0.04 # surface gap between the paired bars, not a border - ys = list(range(len(order)))[::-1] - - for y, bench in zip(ys, order): - g = gem[bench] - ax.barh(y + (h + gap) / 2, g, height=h, color=GEMINI, zorder=3) - ax.annotate( - f"{g:.1f}", - xy=(g, y + (h + gap) / 2), - xytext=(6, 0), - textcoords="offset points", - va="center", - fontsize=10, - fontfamily=MONO, - color=INK, - fontweight="bold", - ) - - if bench in ds: - d = ds[bench] - ax.barh(y - (h + gap) / 2, d, height=h, color=DEEPSEEK, zorder=3) - ax.annotate( - f"{d:.1f}", - xy=(d, y - (h + gap) / 2), - xytext=(6, 0), - textcoords="offset points", - va="center", - fontsize=10, - fontfamily=MONO, - color=INK, - fontweight="bold", - ) - else: - # No bar, and say why — an empty row must never read as a zero score. - ax.annotate( - f"— {pending_note(bench)}", - xy=(0, y - (h + gap) / 2), - xytext=(4, 0), - textcoords="offset points", - va="center", - fontsize=9.5, - fontfamily=MONO, - color=DEEPSEEK, - style="italic", - ) - - ax.set_yticks(ys) - ax.set_yticklabels( - [DISPLAY[b] for b in order], fontsize=11.5, fontfamily=MONO, color=INK - ) - ax.set_xlim(0, 108) - ax.set_xticks([0, 20, 40, 60, 80, 100]) - ax.set_xticklabels( - ["0", "20", "40", "60", "80", "100"], - fontsize=9.5, - fontfamily=MONO, - color=INK_MUTED, - ) - ax.xaxis.grid(True, color=GRID, linewidth=0.8, zorder=0) # solid hairline - ax.set_axisbelow(True) - for side in ("top", "right", "left"): - ax.spines[side].set_visible(False) - ax.spines["bottom"].set_color(GRID) - ax.tick_params(axis="y", length=0) - ax.tick_params(axis="x", length=0) - - # Separator under the comparable block — the rows below it are not a contest. - if comparable and others: - ax.axhline(len(order) - len(comparable) - 0.5, color=GRID, linewidth=1.2) - - ax.legend( - handles=[ - plt.Rectangle( - (0, 0), - 1, - 1, - color=GEMINI, - label=f"{GEMINI_COL} — from repo CSV (team's runs)", - ), - plt.Rectangle( - (0, 0), 1, 1, color=DEEPSEEK, label="DeepSeek-V4-Flash — MEASURED here" - ), - ], - loc="lower center", - bbox_to_anchor=(0.5, -0.175), - ncol=2, - frameon=False, - prop={"family": MONO, "size": 10}, - labelcolor=[GEMINI, DEEPSEEK], - ) - - fig.text( - 0.008, - 0.965, - "Gemini-3.5-flash vs DeepSeek-V4-Flash", - fontsize=15, - fontfamily=MONO, - color=INK, - ) - fig.text( - 0.008, - 0.925, - f"higher is better · only the {len(comparable)} row(s) above the line are a like-for-like comparison", - fontsize=10, - fontfamily=MONO, - color=INK_MUTED, - ) - fig.text( - 0.008, - 0.035, - "DeepSeek V4 Flash is text-only (probed: image -> HTTP 400 'does not support image inputs'; audio -> 400). Those rows have no bar, not a zero.\n" - "Thinking settings differ and are NOT harmonised: Gemini GPQA/MMMLU were run thinking ON; our DeepSeek runs use reasoning_effort=none (its floor).\n" - "Spider-2.0-Lite for DeepSeek is 27.4% after fixing an unclosed-markdown-fence bug in the SQL extractor; before the fix the same responses scored 15.6%.", - fontsize=8.5, - fontfamily=MONO, - color=INK_MUTED, - linespacing=1.7, - ) - - fig.subplots_adjust(left=0.16, right=0.97, top=0.88, bottom=0.26) - fig.savefig(OUT, dpi=200, facecolor=SURFACE) - print(f"wrote {OUT}\n") - print(f"{'benchmark':18} {GEMINI_COL:>18} {'deepseek-v4-flash':>18}") - for b in order: - d = f"{ds[b]:.2f}" if b in ds else f"n/a ({pending_note(b)})" - print(f" {DISPLAY[b]:16} {gem[b]:18.2f} {d:>18}") - - -if __name__ == "__main__": - main() diff --git a/scripts/plot_inkling_radar.py b/scripts/plot_inkling_radar.py deleted file mode 100644 index 2a09ef1..0000000 --- a/scripts/plot_inkling_radar.py +++ /dev/null @@ -1,379 +0,0 @@ -""" -Radar chart of the full Inkling benchmark suite, with Inkling-Small alongside. - -Provenance is the whole point of this chart's design, so it is encoded three ways -(colour, line style, and a per-point tag) and never by colour alone: - - MEASURED — Inkling, run by this repo. Solid line, filled markers. - pub — Inkling-Small, Thinking Machines' published figure. Not run here. - est — Inkling-Small, ESTIMATED by inference from the published table. - Not a measurement of any kind. - -Inkling-Small is not on Fireworks serverless (dedicated deployment only), so -there is nothing measured to plot for it. - - uv run python scripts/plot_inkling_radar.py -""" - -import json -from pathlib import Path - -import matplotlib - -matplotlib.use("Agg") -import matplotlib.pyplot as plt -import numpy as np -from matplotlib.lines import Line2D -from matplotlib.patches import Polygon - -ROOT = Path(__file__).resolve().parent.parent -RESULTS = ROOT / "results" -OUT = RESULTS / "inkling_benchmarks_radar.png" - -# Validated palette (references/palette.md), light mode. Both slots pass every -# check vs this surface (worst adjacent CVD dE 24.7). -SURFACE = "#fcfcfb" -MEASURED = "#2a78d6" # slot 1 — blue -CLAIMED = "#eb6834" # slot 2 — orange -INK = "#0b0b0b" -INK_MUTED = "#898781" -GRID = "#e1e0d9" -MONO = ["DejaVu Sans Mono", "Menlo", "monospace"] - -# Thinking Machines' published table (thinkingmachines.ai/news/inkling-small/), -# reported at effort=0.99 — MAX thinking, not the floor our runs used. -PUBLISHED_SMALL = {"GPQA Diamond": 88.3, "MMMU-Pro": 73.1} - -# Where nothing is published, an explicit estimate. Rationale from the published -# table's own shape: Inkling-Small holds parity on short-form reasoning and -# vision (GPQA +1.1, MMMU-Pro -0.4) but drops hard on factual recall (SimpleQA -# 20.9 vs 43.9) and long-horizon agentic work (Tau-3 Banking 13.6 vs 23.7, -# Terminal-Bench 52.7 vs 63.8). -# MMMLU — scaled by the published Global-MMLU-Lite ratio (86.8/88.7). -# VoxPopuli — audio is near parity (VoiceBench 90.0 vs 91.4, MMAU 77.5/77.2). -# OCR/RefCOCO — perception-bound like MMMU-Pro, so a small markdown. -# Spider — long unseen schema behaves like the agentic cluster: marked down. -ESTIMATE_RATIOS = { - "MMMLU": 86.8 / 88.7, - "VoxPopuli-AA": 0.994, - "OCRBench V2": 0.97, - "olmOCR": 0.97, - "RefCOCO": 0.96, - "Spider-2.0-Lite": 0.865, -} - - -def pct(x): - return round(x * 100, 2) - - -def load_measured(): - """Every axis with a COMPLETE measured run, north then clockwise.""" - - def j(name): - p = RESULTS / name - return json.loads(p.read_text()) if p.exists() else None - - ocr = j("ocrbench_v2_fireworks_inkling_metrics.json") - gpqa = j("fireworks_inkling_thinkingoff_gpqa_diamond_metrics.json") - mmmu = j( - "mmmupro_standard_fireworks_accounts-fireworks-models-inkling_reasoningoff_metrics.json" - ) - mmmlu = j( - "mmmlulite_fireworks_accounts-fireworks-models-inkling_reasoningoff_metrics.json" - ) - spider = j("spider2_lite_local_fireworks_inkling_metrics.json") - asr = j("voxpopuli_aa_fireworks_accounts_fireworks_models_inkling_metrics.json") - rc = j( - "refcoco_val_fireworks_accounts_fireworks_models_inkling_oracle_metrics.json" - ) - - rows = [ - ( - "OCRBench V2", - "Native OCR (EN)", - pct(ocr["en_overall"]) if ocr else None, - 10000 if ocr else 0, - ), - # olmOCR-bench writes its score to stdout, not a metrics file — see - # OLMOCR_SCORE below. - ("olmOCR", "Complex document processing", OLMOCR_SCORE, 1403), - ( - "RefCOCO", - "Object detection (oracle)", - pct(rc["accuracy"]) if rc else None, - rc["total"] if rc else 0, - ), - ( - "Spider-2.0-Lite", - "Text-to-SQL", - pct(spider["accuracy_of_local_135"]), - spider["total_local_subset"], - ), - ( - "VoxPopuli-AA", - "Speech recognition (1-WER)", - pct(1 - asr["corpus_wer"]), - asr.get("samples") or asr.get("num_samples"), - ), - ( - "MMMLU", - "Multilingual Q&A", - pct(mmmlu["macro_accuracy"]), - mmmlu["num_samples"], - ), - ( - "GPQA Diamond", - "PhD-level problem solving", - pct(gpqa["accuracy"]), - gpqa["total"], - ), - ( - "MMMU-Pro", - "Multimodal understanding (std)", - pct(mmmu["accuracy"]), - mmmu["num_samples"], - ), - ] - return [r for r in rows if r[2] is not None] - - -# olmOCR-bench prints its headline to stdout rather than persisting a metrics -# file, so it is pinned here from the run log (1403/1403 pages, 8413 tests). -OLMOCR_SCORE = 74.9 - - -def main(): - scores = load_measured() - labels = [s[0] for s in scores] - subs = [s[1] for s in scores] - vals = [s[2] for s in scores] - ns = [s[3] for s in scores] - n = len(scores) - - small_vals, small_tags = [], [] - for name, v in zip(labels, vals): - if name in PUBLISHED_SMALL: - small_vals.append(PUBLISHED_SMALL[name]) - small_tags.append("pub") - else: - small_vals.append(round(v * ESTIMATE_RATIOS[name], 1)) - small_tags.append("est") - - # Guard: a tag per value. If these ever diverge, zip() below would silently - # drop points rather than fail, which is how a mislabelled chart ships. - assert len(small_tags) == len(small_vals) == len(vals), ( - len(small_tags), - len(small_vals), - len(vals), - ) - - theta = [np.pi / 2 - 2 * np.pi * i / n for i in range(n)] - closed_theta = theta + [theta[0]] - - # No set_theta_zero_location — theta is computed absolutely above, so - # matplotlib's default (0 = east, CCW) keeps index 0 due north. - fig = plt.figure(figsize=(14.5, 10.5), facecolor=SURFACE) - ax = fig.add_subplot(111, polar=True, facecolor=SURFACE) - ax.set_ylim(0, 100) - ax.set_axis_off() # draw our own recessive web instead of matplotlib's - - # Web: solid hairline polygons, one shade off the surface (never dashed). - for r in (20, 40, 60, 80, 100): - ax.add_patch( - Polygon( - [(t, r) for t in theta], - closed=True, - fill=False, - edgecolor=GRID, - linewidth=0.8, - zorder=1, - ) - ) - for t in theta: - ax.plot([t, t], [0, 100], color=GRID, linewidth=0.8, zorder=1) - - # Inkling-Small first so the measured series sits on top of it. - ax.plot( - closed_theta, - small_vals + [small_vals[0]], - color=CLAIMED, - linewidth=2, - linestyle=(0, (5, 3)), - zorder=2, - ) - ax.scatter( - theta, - small_vals, - s=80, - facecolors=SURFACE, - edgecolors=CLAIMED, - linewidths=2, - zorder=3, - ) - - ax.fill(closed_theta, vals + [vals[0]], color=MEASURED, alpha=0.13, zorder=4) - ax.plot(closed_theta, vals + [vals[0]], color=MEASURED, linewidth=2, zorder=5) - ax.scatter( - theta, vals, s=90, color=MEASURED, edgecolors=SURFACE, linewidths=2, zorder=6 - ) - - # Direct value labels — every value readable without a tooltip. Offsets run - # along the spoke (outward for the higher series) so the pair stays apart - # even when the values nearly coincide. - for t, v, sv, tag in zip(theta, vals, small_vals, small_tags): - ux, uy = np.cos(t), np.sin(t) - m_dir, s_dir = (16, -18) if v >= sv else (-18, 16) - if abs(uy) >= abs(ux): - m_off, s_off = (ux * m_dir, uy * m_dir), (ux * s_dir, uy * s_dir) - else: - # Horizontal spokes: these labels are wide, so stack them - # perpendicular instead, nudged outward clear of the markers. - m_off, s_off = (ux * 10, 13), (ux * 10, -13) - ax.annotate( - f"{v:.1f}%", - xy=(t, v), - xytext=m_off, - textcoords="offset points", - ha="center", - va="center", - fontsize=10.5, - fontfamily=MONO, - color=MEASURED, - fontweight="bold", - zorder=7, - ) - ax.annotate( - f"{sv:.1f}% {tag}", - xy=(t, sv), - xytext=s_off, - textcoords="offset points", - ha="center", - va="center", - fontsize=9.5, - fontfamily=MONO, - color=CLAIMED, - zorder=7, - ) - - # One scale anchor on an empty diagonal — ring ticks along a spoke collide - # with that axis's label block, and every value is direct-labelled anyway. - ax.annotate( - "100", - xy=(np.pi / 2 - np.pi / n, 100), - xytext=(8, 4), - textcoords="offset points", - ha="left", - va="center", - fontsize=8.5, - fontfamily=MONO, - color=INK_MUTED, - zorder=5, - ) - - # Axis labels outside the web; descriptor and n on one line. - for t, name, sub, ncount in zip(theta, labels, subs, ns): - x, y = np.cos(t), np.sin(t) - ha = "center" if abs(x) < 0.3 else ("left" if x > 0 else "right") - ax.annotate( - name, - xy=(t, 100), - xytext=(x * 26, y * 26 + 7), - textcoords="offset points", - ha=ha, - va="center", - fontsize=12.5, - fontfamily=MONO, - color=INK, - zorder=5, - ) - ax.annotate( - f"{sub} · n={ncount}", - xy=(t, 100), - xytext=(x * 26, y * 26 - 7), - textcoords="offset points", - ha=ha, - va="center", - fontsize=9, - fontfamily=MONO, - color=INK_MUTED, - zorder=5, - ) - - fig.text( - 0.5, - 0.973, - "Inkling vs Inkling-Small — complete Fireworks benchmark suite", - ha="center", - fontsize=15, - fontfamily=MONO, - color=INK, - ) - fig.text( - 0.5, - 0.945, - "higher is better on every axis · rings every 20 points · n = measured run's sample count", - ha="center", - fontsize=9.5, - fontfamily=MONO, - color=INK_MUTED, - ) - - fig.legend( - handles=[ - Line2D( - [], - [], - color=MEASURED, - lw=2, - marker="o", - markersize=8, - markeredgecolor=SURFACE, - label="Inkling — MEASURED here (reasoning_effort=none, the floor)", - ), - Line2D( - [], - [], - color=CLAIMED, - lw=2, - linestyle=(0, (5, 3)), - marker="o", - markersize=8, - markerfacecolor=SURFACE, - markeredgecolor=CLAIMED, - label="Inkling-Small — NOT measured: pub = published, est = estimated", - ), - ], - loc="upper center", - bbox_to_anchor=(0.5, 0.925), - frameon=False, - prop={"family": MONO, "size": 10.5}, - labelcolor=[MEASURED, CLAIMED], - handlelength=3, - borderaxespad=0, - ) - - fig.text( - 0.5, - 0.045, - "Inkling-Small is not on Fireworks serverless (dedicated deployment only) — nothing is measured for it here.\n" - "pub = published at effort=0.99 (MAX thinking), NOT comparable to our floor runs · est = our inference, 6 of 8 axes, treat as hypotheses.\n" - "RefCOCO is the format-tolerant oracle re-score (strict single-parser score is 32.4% — Inkling emits 4+ coordinate conventions).\n" - "MMMU-Pro is the standard setting, the only one published for both (our vision run measured 68.1%; composite 70.7%).", - ha="center", - fontsize=8.5, - fontfamily=MONO, - color=INK_MUTED, - linespacing=1.8, - ) - - fig.subplots_adjust(left=0.23, right=0.77, top=0.80, bottom=0.185) - fig.savefig(OUT, dpi=200, facecolor=SURFACE) - print(f"wrote {OUT}\n") - print(f"{'benchmark':18} {'inkling':>10} {'small':>10} n") - for (name, _s, v, ncount), sv, tag in zip(scores, small_vals, small_tags): - print(f" {name:16} {v:9.2f}% {sv:9.1f}% {tag:4} n={ncount}") - - -if __name__ == "__main__": - main() diff --git a/scripts/run_fireworks_suite.sh b/scripts/run_fireworks_suite.sh deleted file mode 100755 index 25bf882..0000000 --- a/scripts/run_fireworks_suite.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env bash -# Run the whole benchmark suite against one Fireworks-hosted model. -# -# Serialized on purpose: every runner shares one Fireworks account rate limit, -# so running them concurrently just burns retries on 429s. -# -# Usage: -# scripts/run_fireworks_suite.sh # inkling (serverless) -# MODEL=accounts/fireworks/models/inkling-small scripts/run_fireworks_suite.sh -# ONLY=ocrbench,mmmlu scripts/run_fireworks_suite.sh # subset -# SKIP=ocrbench,olmocr scripts/run_fireworks_suite.sh # all but these -# -# The two OCR benchmarks dominate the cost of a full pass (OCRBench v2 is 10,000 -# image samples, olmOCR-bench 1,403 PDF pages), which is why they're separately -# skippable. -# -# Every runner checkpoints, so re-running skips completed samples. Logs land in -# logs/fireworks_/.log. - -set -uo pipefail -cd "$(dirname "$0")/.." - -MODEL="${MODEL:-accounts/fireworks/models/inkling}" -# PROVIDER lets the same suite target another host (e.g. OpenRouter, which is -# the only place inkling-small is reachable). REASONING is the effort passed -# through; note "none" is a floor on Fireworks but a true off on OpenRouter. -PROVIDER="${PROVIDER:-fireworks}" -REASONING="${REASONING:-off}" -export BENCH_OLMOCR_PROVIDER="$PROVIDER" -export BENCH_OPENROUTER_EFFORT="${OPENROUTER_EFFORT:-none}" -export BENCH_OLMOCR_EFFORT="$BENCH_OPENROUTER_EFFORT" -# ASR effort is pinned separately: inkling wants "none" (at "high" it returns -# content=None after ~233s/sample), but some models REJECT "none" outright — -# gemini-3.7-flash 400s with "Reasoning is mandatory ... cannot be disabled". -ASR_EFFORT="${ASR_EFFORT:-none}" -SLUG="$(printf '%s' "${MODEL##*/}" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9' '-' | sed 's/-$//')" -LOG_DIR="logs/${PROVIDER}_${SLUG}" -mkdir -p "$LOG_DIR" - -# RefCOCO split: val is what refcoco.py (the interfaze reference run) uses. -REFCOCO_SPLIT="${REFCOCO_SPLIT:-val}" -ONLY="${ONLY:-}" -SKIP="${SKIP:-}" - -run_step() { - local name="$1"; shift - if [ -n "$ONLY" ] && [[ ",$ONLY," != *",$name,"* ]]; then - echo "[skip] $name (not in ONLY)" - return 0 - fi - if [ -n "$SKIP" ] && [[ ",$SKIP," == *",$name,"* ]]; then - echo "[skip] $name (in SKIP)" - return 0 - fi - local log="$LOG_DIR/$name.log" - echo "=== [$(date +%H:%M:%S)] $name -> $log" - if "$@" >"$log" 2>&1; then - echo "=== [$(date +%H:%M:%S)] $name OK" - else - # Keep going — one benchmark failing shouldn't strand the other eight. - echo "=== [$(date +%H:%M:%S)] $name FAILED (exit $?), see $log" - fi -} - -# Cheapest first so partial results show up early. -# ASR is pinned to effort=none even when the rest of the suite runs higher. -# Measured on inkling-small via OpenRouter: effort=high returns content=None -# after ~233s per sample (every sample would burn 3 retries and fail), while -# effort=none answers in ~1.3s. Transcription gains nothing from thinking. -if [ -n "${TARGET:-}" ]; then - run_step asr uv run python -m src run --benchmark asr --target "$TARGET" -else - echo " [skip] asr: set TARGET= (python -m src list-targets)" -fi - -if [ -n "${TARGET:-}" ]; then - run_step spider2 uv run python -m src run --benchmark spider2 --target "$TARGET" -else - echo " [skip] spider2: set TARGET= (python -m src list-targets)" -fi - -if [ -n "${TARGET:-}" ]; then - R="$([ "$REASONING" = "high" ] && echo high || echo off)" - run_step mmmupro_standard uv run python -m src run --benchmark mmmu_pro \ - --target "$TARGET" --variant standard --reasoning "$R" - run_step mmmupro_vision uv run python -m src run --benchmark mmmu_pro \ - --target "$TARGET" --variant vision --reasoning "$R" -else - echo " [skip] mmmupro: set TARGET= (python -m src list-targets)" -fi - -if [ -n "${TARGET:-}" ]; then - run_step refcoco uv run python -m src run --benchmark refcoco \ - --target "$TARGET" --variant "$REFCOCO_SPLIT" -else - echo " [skip] refcoco: set TARGET= (python -m src list-targets)" -fi - -if [ -n "${TARGET:-}" ]; then - run_step olmocr uv run python -m src run --benchmark olmocr --target "$TARGET" -else - echo " [skip] olmocr: set TARGET= (python -m src list-targets)" -fi - -if [ -n "${TARGET:-}" ]; then - run_step mmmlu uv run python -m src run --benchmark mmmlu --target "$TARGET" \ - --reasoning "$([ "$REASONING" = "high" ] && echo high || echo off)" -else - echo " [skip] mmmlu: set TARGET= (python -m src list-targets)" -fi - -if [ -n "${TARGET:-}" ]; then - run_step ocrbench uv run python -m src run --benchmark ocrbench_v2 --target "$TARGET" -else - echo " [skip] ocrbench: set TARGET= (python -m src list-targets)" -fi - -# Gated dataset — needs `hf auth login` plus accepted terms on the Hub. -# GPQA now runs through the unified CLI; provider/model/reasoning come from the -# target (src/targets.yaml), so set TARGET= (see `list-targets`). -if [ -n "${TARGET:-}" ]; then - run_step gpqa uv run python -m src run --benchmark gpqa --target "$TARGET" \ - --reasoning "$([ "$REASONING" = "high" ] && echo high || echo off)" -else - echo " [skip] gpqa: set TARGET= (python -m src list-targets)" -fi - -# The format-tolerant oracle (upper bound) is now computed inline by the refcoco -# run and stored under submetrics.oracle in its metrics.json — no separate step. - -echo "=== [$(date +%H:%M:%S)] suite finished for $MODEL" diff --git a/scripts/sheet_columns.py b/scripts/sheet_columns.py deleted file mode 100644 index 2e9f2b3..0000000 --- a/scripts/sheet_columns.py +++ /dev/null @@ -1,303 +0,0 @@ -""" -Emit the Inkling / Inkling-Small columns for the "running" tab of the -Interfaze Benchmarks sheet, read straight from results/*_metrics.json. - - uv run python scripts/sheet_columns.py - -Writes results/sheet_inkling_columns.tsv and prints the same thing, in the -sheet's row order, so the two columns can be pasted in as a block. - -A cell is left BLANK where our run is not comparable to that row's stated -configuration — a blank is recoverable, a mismatched number silently corrupts -the leaderboard. Every blank is explained in the NOTES column. -""" - -import json -from pathlib import Path - -ROOT = Path(__file__).resolve().parent.parent -RESULTS = ROOT / "results" -OUT = RESULTS / "sheet_inkling_columns.tsv" - - -def j(name): - p = RESULTS / name - return json.loads(p.read_text()) if p.exists() else None - - -def main(): - ocr = j("ocrbench_v2_fireworks_inkling_metrics.json") - mmmu_std = j( - "mmmupro_standard_fireworks_accounts-fireworks-models-inkling_reasoningoff_metrics.json" - ) - mmmu_vis = j( - "mmmupro_vision_fireworks_accounts-fireworks-models-inkling_reasoningoff_metrics.json" - ) - mmmlu = j( - "mmmlulite_fireworks_accounts-fireworks-models-inkling_reasoningoff_metrics.json" - ) - gpqa = j("fireworks_inkling_thinkingoff_gpqa_diamond_metrics.json") - spider = j("spider2_lite_local_fireworks_inkling_metrics.json") - asr = j("voxpopuli_aa_fireworks_accounts_fireworks_models_inkling_metrics.json") - - mmmu_composite = (mmmu_std["accuracy"] + mmmu_vis["accuracy"]) / 2 * 100 - - # (sheet Benchmark label, Inkling cell, Inkling Small cell, note) - rows = [ - ( - "OCRBench V2", - f"{ocr['en_overall'] * 100:.2f}", - "", - ( - "Inkling = EN overall, n=10000. Sheet row says Thinking OFF; Inkling " - "cannot disable thinking, this is reasoning_effort=none (its floor). " - "Small: not measured." - ), - ), - ( - "olmOCR", - "74.90", - "", - "1403/1403 pages, 8413 tests, +/-1.1. Small: not measured.", - ), - ( - "RefCoco", - "", - "", - ( - "BLANK ON PURPOSE: this row is TestA; our run is val " - f"(strict {32.37:.2f}, format-tolerant oracle {81.19:.2f}). Not the " - "same split — say the word and I'll run TestA to fill it properly." - ), - ), - ( - "VoxPoppuliCleaned-AA", - f"{(1 - asr['corpus_wer']) * 100:.2f}", - "", - ( - "Reported as 1-WER to match this row's scale (corpus WER 4.61%, " - "n=628). Small: not measured." - ), - ), - ( - "TheSOB", - "", - "", - "BLANK: TheSOB is not implemented in this repo, so neither model was run.", - ), - ( - "Spider-2.0-lite", - f"{spider['accuracy_of_local_135'] * 100:.2f}", - "", - ( - "64/135 SQLite subset. 5 of the 71 misses hit a 120s per-query cap " - "added to the scorer. Small: not measured." - ), - ), - ( - "GPQA-Diamond", - f"{gpqa['accuracy'] * 100:.2f}", - "88.30", - ( - "CONFIG MISMATCH: this row is Thinking On; our Inkling run is the " - "floor (mean 6537 reasoning tokens/sample even so). Small 88.30 is " - "Thinking Machines' PUBLISHED figure at effort=0.99, NOT our measurement." - ), - ), - ( - "MMMLU", - f"{mmmlu['macro_accuracy'] * 100:.2f}", - "", - ( - "CONFIG MISMATCH: row is Thinking On; ours is the floor. Macro over " - "14 langs, n=19950. Small: not measured." - ), - ), - ( - "MMMU-Pro", - f"{mmmu_composite:.2f}", - "73.10", - ( - f"Inkling = standard+vision composite ({mmmu_std['accuracy'] * 100:.2f}" - f" / {mmmu_vis['accuracy'] * 100:.2f}), the published convention. Small " - "73.10 is the PUBLISHED standard-setting figure, not measured, and not " - "a composite." - ), - ), - ] - - lines = ["Benchmark\tInkling\tInkling Small\tNOTES"] - lines += ["\t".join(r) for r in rows] - OUT.write_text("\n".join(lines) + "\n") - - w = max(len(r[0]) for r in rows) - print(f"{'Benchmark'.ljust(w)} {'Inkling':>8} {'Inkling Small':>13}") - print("-" * (w + 27)) - for label, ink, small, _ in rows: - print(f"{label.ljust(w)} {ink or '(blank)':>8} {small or '(blank)':>13}") - print(f"\nwrote {OUT}") - - -# --------------------------------------------------------------------------- -# Extra tabs. The "running" tab above is one column per model; these tabs are -# one ROW per model, with sub-scores as columns. Emitted as separate TSV blocks -# so each can be pasted as a row into its own tab. -# --------------------------------------------------------------------------- - -OUT_TABS = RESULTS / "sheet_inkling_tabs.tsv" - - -def emit_tabs(): - ocr = j("ocrbench_v2_fireworks_inkling_metrics.json") - gpqa = j("fireworks_inkling_thinkingoff_gpqa_diamond_metrics.json") - mmmlu = j( - "mmmlulite_fireworks_accounts-fireworks-models-inkling_reasoningoff_metrics.json" - ) - spider = j("spider2_lite_local_fireworks_inkling_metrics.json") - en = ocr["en_scores"] - - def p(x): - return f"{x * 100:.2f}" - - blocks = [] - - # --- OCRBench V2 tab: model | Avg Real | Recog | Refer | Spot | Extract - # | Parse | Calc | Understand | Reason (EN scores; matches the tab's - # Interfaze row, whose Avg Real equals the running-tab OCRBench value) - blocks.append( - ( - "OCRBench V2 tab", - [ - "model", - "Avg Real", - "Recog", - "Refer", - "Spot", - "Extract", - "Parse", - "Calc", - "Understand", - "Reason", - ], - [ - "Inkling", - p(ocr["en_overall"]), - p(en["text_recognition"]["avg"]), - p(en["text_detection"]["avg"]), - p(en["text_spotting"]["avg"]), - p(en["relationship_extraction"]["avg"]), - p(en["element_parsing"]["avg"]), - p(en["mathematical_calculation"]["avg"]), - p(en["visual_text_understanding"]["avg"]), - p(en["knowledge_reasoning"]["avg"]), - ], - ) - ) - - # --- olmOCR tab. olmOCR-bench prints its per-split table to stdout instead - # of persisting a metrics file, so these come from the run log - # (logs/fireworks_inkling/olmocr.log, 1403/1403 pages, 8413 tests). - blocks.append( - ( - "olmOCR tab", - [ - "model", - "Overall", - "ArXiv", - "OldScansMath", - "Tables", - "OldScans", - "Headers", - "MultiCol", - "LongTinyText", - "Base", - "Real Overall", - ], - [ - "Inkling", - "74.90", - "71.20", - "78.80", - "82.60", - "41.60", - "91.60", - "72.40", - "61.80", - "99.10", - "74.9±1.1", - ], - ) - ) - - # --- GPQA tab: model | Overall | Physics (n=86) | Chemistry (n=93) | Biology (n=19) - d = gpqa["per_domain"] - blocks.append( - ( - "GPQA tab", - [ - "model", - "Overall", - "Physics (n=86)", - "Chemistry (n=93)", - "Biology (n=19)", - ], - [ - "Inkling", - p(gpqa["accuracy"]), - p(d["Physics"]["accuracy"]), - p(d["Chemistry"]["accuracy"]), - p(d["Biology"]["accuracy"]), - ], - ) - ) - - # --- MMMLU tab: languages are ROWS and models are COLUMNS, so this is a - # column (macro first, then the 14 languages in the sheet's order). - order = [ - "FR_FR", - "PT_BR", - "BN_BD", - "JA_JP", - "DE_DE", - "YO_NG", - "ES_LA", - "ID_ID", - "ZH_CN", - "SW_KE", - "IT_IT", - "AR_XY", - "KO_KR", - "HI_IN", - ] - pl = mmmlu["per_language"] - blocks.append(("MMMLU tab (column: macro then langs)", ["lang", "Inkling"], None)) - mmmlu_rows = [["macro", p(mmmlu["macro_accuracy"])]] - mmmlu_rows += [[lg, p(pl[lg]["accuracy"])] for lg in order if lg in pl] - - # --- Spider-2.0 tab: Model | Spider-2.0-lite - blocks.append( - ( - "Spider-2.0 tab", - ["Model", "Spider-2.0-lite"], - ["Inkling", p(spider["accuracy_of_local_135"])], - ) - ) - - lines = [] - for title, header, row in blocks: - lines.append(f"# {title}") - lines.append("\t".join(header)) - if row: - lines.append("\t".join(row)) - else: - lines += ["\t".join(r) for r in mmmlu_rows] - lines.append("") - OUT_TABS.write_text("\n".join(lines) + "\n") - print("\n".join(lines)) - print(f"wrote {OUT_TABS}") - - -if __name__ == "__main__": - main() - print() - emit_tabs()