Feat/bench core abstraction

.github/workflows/add-target.yml

@@ -0,0 +1,61 @@
+name: add-target
+
+on:
+  workflow_dispatch:
+    inputs:
+      name:
+        description: "target name (map key, e.g. my-model)"
+        required: true
+      provider:
+        description: "provider"
+        required: true
+        type: choice
+        options: [fireworks, openrouter, openai, anthropic, gemini, interfaze]
+      model_id:
+        description: "provider model id (e.g. accounts/fireworks/models/my-model)"
+        required: true
+      capabilities_json:
+        description: 'optional capability overrides as JSON, e.g. {"reasoning":{"style":"thinking_budget","off_value":0,"on_value":-1,"true_off":true}}'
+        required: false
+        default: ""
+      ci_regression:
+        description: "include in the nightly regression run"
+        type: boolean
+        default: false
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  add:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v9.0.0
+        with:
+          enable-cache: true
+      - run: uv sync
+      - name: Validate + append target
+        env:
+          T_NAME: ${{ github.event.inputs.name }}
+          T_PROVIDER: ${{ github.event.inputs.provider }}
+          T_MODEL: ${{ github.event.inputs.model_id }}
+          T_CAPS: ${{ github.event.inputs.capabilities_json }}
+          T_CI: ${{ github.event.inputs.ci_regression }}
+        run: |
+          ARGS=(--name "$T_NAME" --provider "$T_PROVIDER" --model-id "$T_MODEL" --capabilities-json "$T_CAPS")
+          [ "$T_CI" = "true" ] && ARGS+=(--ci-regression)
+          uv run python scripts/add_target.py "${ARGS[@]}"
+      - name: Open PR
+        uses: peter-evans/create-pull-request@v7
+        with:
+          branch: add-target/${{ github.event.inputs.name }}
+          title: "Add benchmark target: ${{ github.event.inputs.name }}"
+          commit-message: "feat(targets): add ${{ github.event.inputs.name }}"
+          body: |
+            Adds `${{ github.event.inputs.name }}` (${{ github.event.inputs.provider }} / `${{ github.event.inputs.model_id }}`) to `src/targets.yaml`.
+
+            The entry was validated (loads, resolves capabilities, builds its adapter).
+            **Merging this PR triggers the benchmark run for the new target** (benchmark.yml push trigger).
+          add-paths: src/targets.yaml

.github/workflows/benchmark.yml

@@ -0,0 +1,137 @@
+name: benchmark
+
+on:
+  workflow_dispatch:
+    inputs:
+      target:
+        description: "target name — required (see src/targets.yaml / list-targets)"
+        required: true
+      benchmarks:
+        description: "comma-separated benchmarks, or 'all' for the CI set"
+        default: all
+      sample_size:
+        description: "samples per benchmark (blank = full run — slow/expensive)"
+        default: "5"
+      reasoning:
+        description: "reasoning override: off|low|medium|high (blank = benchmark default)"
+        default: ""
+  schedule:
+    - cron: "0 6 * * 1"
+  push:
+    branches: [main]
+    paths: [src/targets.yaml] # a merged target change runs a full benchmark for it
+
+# never run two paid benchmark passes for the same ref at once (queue, don't cancel
+# — a half-finished expensive run's spend shouldn't be thrown away)
+concurrency:
+  group: benchmark-${{ github.ref }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  plan:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.m.outputs.matrix }}
+      targets: ${{ steps.m.outputs.targets }}
+      has_work: ${{ steps.m.outputs.has_work }}
+      sample: ${{ steps.m.outputs.sample }}
+      reasoning: ${{ steps.m.outputs.reasoning }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0 # need the merge's parent to diff targets.yaml on push
+      - uses: astral-sh/setup-uv@v9.0.0
+        with:
+          enable-cache: true
+      - run: uv sync
+      - id: m
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          INPUT_TARGET: ${{ github.event.inputs.target }}
+          INPUT_BENCHMARKS: ${{ github.event.inputs.benchmarks }}
+        run: |
+          if [ "${{ github.event_name }}" = "push" ]; then
+            git show "${{ github.event.before }}:src/targets.yaml" > /tmp/old_targets.yaml 2>/dev/null || : > /tmp/old_targets.yaml
+            export OLD_TARGETS_FILE=/tmp/old_targets.yaml
+            SAMPLE=""   # a merged target change gets a full benchmark run
+          else
+            SAMPLE="${{ github.event.inputs.sample_size || '5' }}"
+          fi
+          MATRIX="$(uv run python scripts/ci_matrix.py)"
+          echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT"
+          echo "targets=$(printf '%s' "$MATRIX" | uv run python -c 'import json,sys; print(" ".join(sorted({i["target"] for i in json.load(sys.stdin)["include"]})))')" >> "$GITHUB_OUTPUT"
+          echo "has_work=$(printf '%s' "$MATRIX" | uv run python -c 'import json,sys; print(str(bool(json.load(sys.stdin)["include"])).lower())')" >> "$GITHUB_OUTPUT"
+          echo "sample=$SAMPLE" >> "$GITHUB_OUTPUT"
+          echo "reasoning=${{ github.event.inputs.reasoning }}" >> "$GITHUB_OUTPUT"
+
+  run:
+    needs: plan
+    if: needs.plan.outputs.has_work == 'true' # no target selected/flagged -> nothing runs
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+      FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
+      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+      INTERFAZE_API_KEY: ${{ secrets.INTERFAZE_API_KEY }}
+      HF_TOKEN: ${{ secrets.HF_TOKEN }}
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v9.0.0
+        with:
+          enable-cache: true
+      - run: uv sync
+      - name: Run ${{ matrix.benchmark }} on ${{ matrix.target }}
+        run: |
+          SAMPLE="${{ needs.plan.outputs.sample }}"
+          REASONING="${{ needs.plan.outputs.reasoning }}"
+          uv run python -m src run \
+            --target "${{ matrix.target }}" --benchmark "${{ matrix.benchmark }}" \
+            ${SAMPLE:+--sample "$SAMPLE"} ${REASONING:+--reasoning "$REASONING"}
+      - name: Upload metrics
+        uses: actions/upload-artifact@v4
+        with:
+          name: metrics-${{ matrix.target }}-${{ matrix.benchmark }}
+          path: results/${{ matrix.benchmark }}*/${{ matrix.target }}/metrics.json
+          if-no-files-found: ignore
+
+  report:
+    needs: [plan, run]
+    if: always() && needs.plan.outputs.has_work == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v9.0.0
+        with:
+          enable-cache: true
+      - run: uv sync
+      - name: Fetch run metrics
+        uses: actions/download-artifact@v4
+        with:
+          path: incoming
+          merge-multiple: true
+      - name: Overlay fresh metrics onto the contract
+        run: |
+          # artifacts store their original results/<bench>/<target>/metrics.json path
+          if [ -d incoming/results ]; then cp -r incoming/results/* results/; fi
+      - name: Compare vs baseline + write summary
+        run: |
+          for t in ${{ needs.plan.outputs.targets }}; do
+            uv run python scripts/ci_compare.py --target "$t" >> "$GITHUB_STEP_SUMMARY"
+          done
+          {
+            echo ""
+            echo '<details><summary>Full score tables</summary>'
+            echo ""
+            echo '```'
+            uv run python scripts/report_scores.py --all 2>&1 || true
+            echo '```'
+            echo '</details>'
+          } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/tests.yml

@@ -0,0 +1,28 @@
+name: tests
+
+on:
+  push:
+    branches: [main] # post-merge sanity; feature branches are covered by their PR
+  pull_request:
+
+concurrency:
+  group: tests-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v9.0.0
+        with:
+          enable-cache: true
+      - name: Install deps
+        run: uv sync --group dev
+      - name: Lint (new code)
+        run: uv run ruff check src tests
+      - name: Tests (offline)
+        run: uv run pytest -q

.gitignore

@@ -208,3 +208,18 @@ __marimo__/
 
 # olmOCR-Bench dataset + per-candidate prediction outputs (large, downloaded from HF)
 benchmarks/olmocr_bench/olmocr_bench/
+
+# Downloaded benchmark datasets (large; fetched at runtime, not source)
+benchmarks/spider2_lite/data/
+benchmarks/olmocr/bench/full_data/
+
+# Benchmark outputs are never tracked — they are run artifacts, not source, and
+# large (hundreds of MB incl. Spider2/OCRBench responses). Kept on disk locally;
+# regenerate with a run.
+results/
+
+# OCRBench spotting-eval scratch (generated during scoring)
+benchmarks/ocrbench_v2/eval_scripts/spotting_eval/gt/
+benchmarks/ocrbench_v2/eval_scripts/spotting_eval/submit/
+benchmarks/ocrbench_v2/eval_scripts/spotting_eval/*.zip
+benchmarks/ocrbench_v2/results.zip